From 70ece3501f5be4d5ed20e99958f2bc37183878a8 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 14:55:28 -0700 Subject: [PATCH 001/235] fix(proxy): price a billed failure row at the deployment's long-context tier like the success path (#47647) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 61 ++++++++++--------- ..._utils_model_creation_and_error_logging.py | 29 +++++++++ 2 files changed, 61 insertions(+), 29 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 20a0d4d5053..147f9292a21 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -91,7 +91,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse -from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, CostPerToken, ModelInfo, Usage +from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage try: from litellm_enterprise.enterprise_callbacks.send_emails.base_email import ( @@ -159,6 +159,7 @@ from litellm.litellm_core_utils.core_helpers import ( from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int from litellm.litellm_core_utils.litellm_logging import Logging, deployment_pricing_model_info +from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.litellm_core_utils.served_output_texts import ( @@ -1193,33 +1194,33 @@ def _billed_earlier_attempts( ) -def _deployment_cost_per_token(deployment_id: str | None, model: str) -> CostPerToken | None: - """The pricing the router registered for this deployment under its ``model_info.id``, read - through ``deployment_pricing_model_info`` exactly as the success path does; None means the - public cost map. The id is ``Logging.get_router_model_id()`` (the logging object's own - ``litellm_params``), never the flattened ``model_call_details`` copy a request body key can - overwrite.""" - model_info: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model) - if model_info is None: - return None - return CostPerToken( - input_cost_per_token=model_info["input_cost_per_token"] or 0.0, - output_cost_per_token=model_info["output_cost_per_token"] or 0.0, - ) +def _deployment_priced_input_cost( + model: str, custom_llm_provider: str | None, prompt_tokens: int, deployment_pricing: ModelInfo +) -> float: + """Input price for the estimated prompt at the deployment's registered rates, read through + ``deployment_pricing_model_info`` and priced by ``generic_cost_per_token`` the way the success + path prices them, so a long-context tier the deployment declares (``input_cost_per_token_above_200k_tokens``) + is honoured instead of flattened to the standard rate. An entry it cannot price keeps the row at $0.""" + usage: Final = Usage(prompt_tokens=prompt_tokens, completion_tokens=0, total_tokens=prompt_tokens) + try: + prompt_cost, _ = generic_cost_per_token( + model=model, + usage=usage, + custom_llm_provider=custom_llm_provider or str(deployment_pricing.get("litellm_provider") or ""), + model_info=deployment_pricing, + ) + except Exception as exc: # noqa: BLE001 # a malformed registered entry must not drop the failure row + verbose_proxy_logger.debug("failure spend estimate: could not price deployment of %s: %s", model, exc) + return 0.0 + return prompt_cost -def _priced_input_cost( - pricing_model: str, - custom_llm_provider: str | None, - prompt_tokens: int, - custom_cost_per_token: CostPerToken | None, -) -> float | None: +def _priced_input_cost(pricing_model: str, custom_llm_provider: str | None, prompt_tokens: int) -> float | None: try: estimated_input_cost, _ = litellm.cost_per_token( model=pricing_model, custom_llm_provider=custom_llm_provider, prompt_tokens=prompt_tokens, - custom_cost_per_token=custom_cost_per_token, ) except Exception as exc: # noqa: BLE001 # cost_per_token raises BadRequestError for an unmapped model and the row must still be written verbose_proxy_logger.debug("failure spend estimate: could not price %s: %s", pricing_model, exc) @@ -1233,18 +1234,20 @@ def _estimated_input_cost( deployment_id: str | None, base_model: str | None, ) -> float: - """Input price for the estimated prompt: the deployment's registered rate when it has one, else - the cost map's rate for ``base_model``, else the deployment model, the same candidate order - ``completion_cost`` walks (an Azure deployment name is not in the cost map, its ``base_model`` - is; a ``base_model`` the cost map does not know falls through to the deployment model). A model - no candidate can price keeps the row at $0 rather than dropping the failure row.""" + """Input price for the estimated prompt: the deployment's registered rates when it has any (the id + is ``Logging.get_router_model_id()``, never the flattened ``model_call_details`` copy a request body + key can overwrite), else the cost map's rate for ``base_model``, else the deployment model, the same + candidate order ``completion_cost`` walks (an Azure deployment name is not in the cost map, its + ``base_model`` is; a ``base_model`` the cost map does not know falls through to the deployment + model). A model no candidate can price keeps the row at $0 rather than dropping the failure row.""" model: Final = str(model_call_details.get("model") or "") custom_llm_provider: Final = str(model_call_details.get("custom_llm_provider") or "") or None - custom_cost_per_token: Final = _deployment_cost_per_token(deployment_id, model) + deployment_pricing: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model) + if deployment_pricing is not None: + return _deployment_priced_input_cost(model, custom_llm_provider, prompt_tokens, deployment_pricing) pricing_models: Final = (base_model, model) if base_model is not None else (model,) candidate_costs: Final = ( - _priced_input_cost(pricing_model, custom_llm_provider, prompt_tokens, custom_cost_per_token) - for pricing_model in pricing_models + _priced_input_cost(pricing_model, custom_llm_provider, prompt_tokens) for pricing_model in pricing_models ) return next((cost for cost in candidate_costs if cost is not None), 0.0) diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 59e29c9b33c..48ad3490ebb 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2160,6 +2160,35 @@ def test_a_timed_out_request_is_priced_at_the_rate_the_router_registered_for_the assert response_cost == pytest.approx(usage.prompt_tokens * 0.25) +@pytest.mark.parametrize(("prompt_tokens", "expected_rate"), [(1_000, 1e-7), (250_000, 5e-7)]) +def test_a_timed_out_request_is_priced_at_the_deployments_long_context_tier(monkeypatch, prompt_tokens, expected_rate): + """A deployment that declares ``input_cost_per_token_above_200k_tokens`` next to its standard rate + bills a 250k-token prompt at the tier on the success path; the failure row must not flatten it to + the standard rate.""" + from litellm.proxy.utils import _estimated_input_cost + + monkeypatch.setitem( + litellm.model_cost, + "dep-lit-f9-tiered", + { + "litellm_provider": "openai", + "mode": "chat", + "input_cost_per_token": 1e-7, + "input_cost_per_token_above_200k_tokens": 5e-7, + "output_cost_per_token": 4e-7, + }, + ) + + response_cost = _estimated_input_cost( + model_call_details={"model": FAILURE_USAGE_MODEL, "custom_llm_provider": "openai"}, + prompt_tokens=prompt_tokens, + deployment_id="dep-lit-f9-tiered", + base_model=None, + ) + + assert response_cost == pytest.approx(prompt_tokens * expected_rate) + + def test_every_earlier_timed_out_attempt_is_billed_at_its_own_deployment_rate(monkeypatch): """The router leaves one RetryAttemptRecord per failed attempt in metadata.previous_models, the final attempt included; the earlier Timeout ones are billed (each at the deployment that served From 95b6eea5bb4857cc29f4702b89cad176b422e820 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 14:59:34 -0700 Subject: [PATCH 002/235] test(perf): tag a UI journey sample with its API response, not a Next.js prefetch under /ui/ (#47650) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/setup/ui_journeys.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/e2e/perf/setup/ui_journeys.py b/tests/e2e/perf/setup/ui_journeys.py index 2b7bda77733..4de97d8c717 100644 --- a/tests/e2e/perf/setup/ui_journeys.py +++ b/tests/e2e/perf/setup/ui_journeys.py @@ -10,8 +10,8 @@ from dataclasses import dataclass from datetime import datetime from itertools import chain from types import MappingProxyType -from urllib.parse import urlsplit from typing import Final +from urllib.parse import urlsplit from playwright.sync_api import Error as PlaywrightError from playwright.sync_api import Locator, Page, Response, expect, sync_playwright @@ -189,7 +189,9 @@ def _login(page: Page, dashboard: Dashboard) -> None: def _is_data_fetch(response: Response) -> bool: - return response.request.resource_type in ("fetch", "xhr") and "/_next/" not in response.url + """API calls only. The Admin UI's own assets and Next.js prefetches live under /ui/ and /_next/.""" + path: Final = urlsplit(response.url).path + return response.request.resource_type in ("fetch", "xhr") and not path.startswith(("/ui/", "/_next/")) def _worst(responses: list[Response]) -> Response | None: From 640f2d865bfa954d37bbbffc447f4e504e1dad21 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 15:01:53 -0700 Subject: [PATCH 003/235] fix(proxy): scope the search free-tier counter to the request's provider (#47646) * fix(proxy): scope the search free-tier counter to the request's provider * fix(proxy): read the provider with get so partial spend payloads still get the free tier * test(proxy): give the grounded spend payload fixture its provider --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/hooks/search_free_tier.py | 8 ++-- .../hooks/test_proxy_track_cost_callback.py | 1 + .../unit/proxy/hooks/test_search_free_tier.py | 40 +++++++++++++------ 3 files changed, 34 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/hooks/search_free_tier.py b/litellm/proxy/hooks/search_free_tier.py index 4ef300dfbac..34f4d50550a 100644 --- a/litellm/proxy/hooks/search_free_tier.py +++ b/litellm/proxy/hooks/search_free_tier.py @@ -34,8 +34,9 @@ def freed_units(allowance: int, units: int, total_after: int) -> int: return max(0, min(units, allowance - (total_after - units))) -def search_free_tier_key(allowance: SearchFreeRequests, now: datetime) -> str: - return f"litellm:search_free_tier:{allowance['shared_group']}:{now.strftime(_PERIOD_STAMP[allowance['period']])}" +def search_free_tier_key(provider: str, allowance: SearchFreeRequests, now: datetime) -> str: + period_stamp: Final = now.strftime(_PERIOD_STAMP[allowance["period"]]) + return f"litellm:search_free_tier:{provider}:{allowance['shared_group']}:{period_stamp}" def search_free_tier_ttl(allowance: SearchFreeRequests, now: datetime) -> int: @@ -107,7 +108,8 @@ async def apply_search_free_tier( "apply_search_free_tier is on but the proxy has no Redis, grounding is billed at full price" ) return response_cost - key: Final = search_free_tier_key(allowance, now) + provider: Final = sl_object["custom_llm_provider"] or model_info["litellm_provider"] + key: Final = search_free_tier_key(provider, allowance, now) try: total_after: Final = await counter.async_increment(key, units, ttl=search_free_tier_ttl(allowance, now)) except Exception: # noqa: BLE001 # any counter failure means full price, never a lost spend row diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index 289f8cd4a1a..ba1b5536a07 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -3202,6 +3202,7 @@ def _grounded_gemini_kwargs(response_cost: float) -> dict: "response_cost": response_cost, "request_tags": None, "cache_hit": None, + "custom_llm_provider": "vertex_ai", "cost_breakdown": { "input_cost": 0.0, "output_cost": 0.0, diff --git a/tests/unit/proxy/hooks/test_search_free_tier.py b/tests/unit/proxy/hooks/test_search_free_tier.py index 3dc89c0dc72..b57908378e5 100644 --- a/tests/unit/proxy/hooks/test_search_free_tier.py +++ b/tests/unit/proxy/hooks/test_search_free_tier.py @@ -31,9 +31,9 @@ class _BrokenCounter: raise ConnectionError("redis is down") -def _model_info(billing_unit: str, count: int, period: str = "month") -> dict: +def _model_info(billing_unit: str, count: int, period: str = "month", provider: str = "vertex_ai") -> dict: return { - "litellm_provider": "vertex_ai", + "litellm_provider": provider, "mode": "chat", "web_search_billing_unit": billing_unit, "search_context_cost_per_query": {"search_context_size_medium": RATE}, @@ -52,6 +52,7 @@ def _call(model_info: dict, response_cost: float, cache_hit: bool | None = None) sl_object = { "response_cost": response_cost, "cache_hit": cache_hit, + "custom_llm_provider": model_info["litellm_provider"], "cost_breakdown": {"input_cost": 0.1, "output_cost": 0.2, "tool_usage_cost": 0.5, "total_cost": response_cost}, "model_map_information": {"model_map_key": "gemini-test", "model_map_value": model_info}, } @@ -73,12 +74,12 @@ def test_freed_units_covers_only_what_landed_under_the_allowance(allowance, unit assert freed_units(allowance, units, total_after) == expected -def test_counter_key_is_per_group_and_utc_period(): - assert search_free_tier_key({"count": 1, "period": "month", "shared_group": "g3"}, NOW) == ( - "litellm:search_free_tier:g3:2026-10" +def test_counter_key_is_per_provider_group_and_utc_period(): + assert search_free_tier_key("gemini", {"count": 1, "period": "month", "shared_group": "g3"}, NOW) == ( + "litellm:search_free_tier:gemini:g3:2026-10" ) - assert search_free_tier_key({"count": 1, "period": "day", "shared_group": "g2"}, NOW) == ( - "litellm:search_free_tier:g2:2026-10-20" + assert search_free_tier_key("vertex_ai", {"count": 1, "period": "day", "shared_group": "g2"}, NOW) == ( + "litellm:search_free_tier:vertex_ai:g2:2026-10-20" ) @@ -107,8 +108,23 @@ async def test_per_query_allowance_is_shared_across_calls_and_split_on_the_cross assert c2 == k2 == s2 == pytest.approx(1.0 - 2 * RATE) assert b2["discount_amount"] == pytest.approx(2 * RATE) assert c3 == k3 == s3 == 1.0 and "discount_amount" not in b3 - assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 15} - assert counter.ttls["litellm:search_free_tier:test-group:2026-10"] > 0 + assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 15} + assert counter.ttls["litellm:search_free_tier:vertex_ai:test-group:2026-10"] > 0 + + +@pytest.mark.asyncio +async def test_gemini_api_and_vertex_ai_grounding_draw_on_separate_allowances(): + counter = _Counter() + gemini_api = _model_info("per_query", count=3, provider="gemini") + vertex_ai = _model_info("per_query", count=3, provider="vertex_ai") + kwargs, sl_object = _call(gemini_api, response_cost=1.0) + await apply_search_free_tier(kwargs, sl_object, 1.0, _response(2), counter, NOW) + kwargs, sl_object = _call(vertex_ai, response_cost=1.0) + vertex_cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(4), counter, NOW) + kwargs, sl_object = _call(gemini_api, response_cost=1.0) + gemini_cost = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(1), counter, NOW) + assert vertex_cost == pytest.approx(1.0 - 3 * RATE), "vertex ai had its whole allowance left" + assert gemini_cost == pytest.approx(1.0 - RATE), "gemini api had one free query left" @pytest.mark.asyncio @@ -121,7 +137,7 @@ async def test_per_prompt_allowance_counts_a_grounded_prompt_once_whatever_the_q second = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) assert first == pytest.approx(1.0 - RATE) assert second == 1.0 - assert counter.totals == {"litellm:search_free_tier:test-group:2026-10-20": 2} + assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10-20": 2} @pytest.mark.asyncio @@ -172,7 +188,7 @@ async def test_a_second_callback_run_on_the_same_request_counts_it_once(): first = await apply_search_free_tier(kwargs, sl_object, 1.0, _response(5), counter, NOW) second = await apply_search_free_tier(kwargs, sl_object, first, _response(5), counter, NOW) assert first == second == pytest.approx(1.0 - 5 * RATE) - assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5} + assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 5} @pytest.mark.asyncio @@ -191,4 +207,4 @@ async def test_freed_cost_never_exceeds_the_billed_cost(): kwargs, sl_object = _call(_model_info("per_query", count=100), response_cost=billed) assert await apply_search_free_tier(kwargs, sl_object, billed, _response(5), counter, NOW) == 0.0 assert sl_object["cost_breakdown"]["discount_amount"] == pytest.approx(billed) - assert counter.totals == {"litellm:search_free_tier:test-group:2026-10": 5} + assert counter.totals == {"litellm:search_free_tier:vertex_ai:test-group:2026-10": 5} From 9026f42710c3377c6a054f2fe57bf3c8f3d409a6 Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Tue, 20 Oct 2026 15:04:26 -0700 Subject: [PATCH 004/235] fix(ui): offer access-group models in team per-model rate limit and budget pickers (#47630) The team Model-Specific Rate Limits picker built its options from the team's direct models only. A team that gets its models through access groups has models ["no-default-models"], so the sentinel was the only option, and a saved {"no-default-models": N} limit never enforces since the limiter matches the requested model name. Build one option list for every team per-model field from direct plus access-group models, expand wildcards, treat an empty list as all models, and drop sentinels and wildcard patterns, which no limit or budget lookup ever matches. The team per-model budget field shared the broken list and is fixed with it. LIT-9490 --- .../team/TeamInfo.integration.test.tsx | 41 +++++++++++++++++++ .../src/components/team/TeamInfo.tsx | 25 ++++------- .../components/team/teamModelAccess.test.ts | 39 +++++++++++++++++- .../src/components/team/teamModelAccess.ts | 20 +++++++++ 4 files changed, 106 insertions(+), 19 deletions(-) diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.integration.test.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.integration.test.tsx index 7f567a7f936..fc054e51e5b 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.integration.test.tsx @@ -816,3 +816,44 @@ describe("TeamInfoView - member budget reset prompt", () => { expect(screen.getByRole("option", { name: "claude-haiku-4-5" })).toBeInTheDocument(); }); }); + +describe("TeamInfoView - model rate limit picker", () => { + const props = { + teamId: "123", + onUpdate: vi.fn(), + onClose: vi.fn(), + accessToken: "test-token", + is_team_admin: true, + is_proxy_admin: true, + editTeam: false, + }; + + beforeEach(() => { + seedDefaultMocks(); + }); + + afterEach(() => { + vi.clearAllMocks(); + }); + + it("offers access-group models, not no-default-models, in the model rate limit picker", async () => { + const user = userEvent.setup({ delay: null }); + vi.mocked(networking.teamInfoCall).mockResolvedValue( + createMockTeamData({ + models: ["no-default-models"], + access_group_models: ["claude-haiku-4-5"], + }) as TeamData, + ); + + renderWithProviders(); + await waitFor(() => expect(screen.queryAllByText("Test Team").length).toBeGreaterThan(0)); + await user.click(screen.getByRole("tab", { name: "Settings" })); + await user.click(await screen.findByRole("button", { name: /edit settings/i })); + await user.click(screen.getByRole("button", { name: /add model limit/i })); + await user.click(screen.getByPlaceholderText("Select model")); + + expect(await screen.findByRole("option", { name: "claude-haiku-4-5" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "no-default-models" })).not.toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "gpt-4o" })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx index 29c048034f4..0d63f5d0833 100644 --- a/ui/litellm-dashboard/src/components/team/TeamInfo.tsx +++ b/ui/litellm-dashboard/src/components/team/TeamInfo.tsx @@ -75,6 +75,7 @@ import { computeTeamModelBadges, normalizeTeamModelSelection, TeamAccessGroupModelGrant, + teamPerModelLimitOptions, TeamModelBadge, TeamModelBadgeKind, } from "./teamModelAccess"; @@ -89,7 +90,6 @@ import ModelAliasManager from "../common_components/ModelAliasManager"; import AgentSelector from "../agent_management/AgentSelector"; import DeleteResourceModal from "../common_components/DeleteResourceModal"; import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; -import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; import GuardrailSettingsView from "../GuardrailSettingsView"; import LoggingSettingsView from "../logging_settings_view"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; @@ -666,20 +666,9 @@ const TeamInfoView: React.FC = ({ [mcpServersLoading || mcpToolsetsLoading || accessGroupsLoading, "the MCP server inventory is still loading"], ] as const ).find(([failed]) => failed)?.[1] ?? null; - const availableRateLimitModels = useMemo(() => { + const perModelLimitModels = useMemo(() => { const selected = (isEditing ? watchedModels : undefined) ?? teamData?.team_info?.models ?? []; - if (selected.includes("all-proxy-models") || selected.includes("all-team-models")) { - return userModels; - } - return unfurlWildcardModelsInList(selected, userModels); - }, [isEditing, watchedModels, teamData, userModels]); - const availableModelBudgetModels = useMemo(() => { - const selected = (isEditing ? watchedModels : undefined) ?? teamData?.team_info?.models ?? []; - if (selected.length === 0 || selected.includes("all-proxy-models") || selected.includes("all-team-models")) { - return userModels; - } - const granted = [...selected, ...(teamData?.team_info?.access_group_models ?? [])]; - return unfurlWildcardModelsInList(Array.from(new Set(granted)), userModels); + return teamPerModelLimitOptions(selected, teamData?.team_info?.access_group_models ?? [], userModels); }, [isEditing, watchedModels, teamData, userModels]); const teamEditAccess = useMemo(() => parseTeamEditAccess(teamData?.team_info?.caller_edit_access), [teamData]); @@ -1724,7 +1713,7 @@ const TeamInfoView: React.FC = ({ premiumUser={premiumUser} value={teamMemberModelMaxBudget} onChange={setTeamMemberModelMaxBudget} - availableModels={availableModelBudgetModels} + availableModels={perModelLimitModels} hint="Set a default per-model spend cap for each team member, with an independent reset window." /> = ({ premiumUser={premiumUser} value={teamModelMaxBudget} onChange={setTeamModelMaxBudget} - availableModels={availableRateLimitModels} + availableModels={perModelLimitModels} usage={info.model_max_budget_usage} hint="Cap this team's spend on individual models, each with its own reset window. Every key on the team shares the cap unless the key sets its own budget for that model." /> @@ -1906,7 +1895,7 @@ const TeamInfoView: React.FC = ({ inputId={id} value={value ?? ""} onValueChange={onChange} - options={availableRateLimitModels.map((model) => ({ + options={perModelLimitModels.map((model) => ({ label: model, value: model, }))} @@ -2644,7 +2633,7 @@ const TeamInfoView: React.FC = ({ ), type: "model-max-budget" as const, - availableModels: availableModelBudgetModels, + availableModels: perModelLimitModels, premiumUser, }, { diff --git a/ui/litellm-dashboard/src/components/team/teamModelAccess.test.ts b/ui/litellm-dashboard/src/components/team/teamModelAccess.test.ts index 3f0b4e20407..5ca42a9391b 100644 --- a/ui/litellm-dashboard/src/components/team/teamModelAccess.test.ts +++ b/ui/litellm-dashboard/src/components/team/teamModelAccess.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { computeTeamModelBadges, normalizeTeamModelSelection, TeamAccessGroupModelGrant } from "./teamModelAccess"; +import { + computeTeamModelBadges, + normalizeTeamModelSelection, + TeamAccessGroupModelGrant, + teamPerModelLimitOptions, +} from "./teamModelAccess"; const GRANTS: TeamAccessGroupModelGrant[] = [ { access_group_id: "ag-1", access_group_name: "shared", models: ["haiku", "gpt-4o-mini"] }, @@ -84,3 +89,35 @@ describe("computeTeamModelBadges", () => { ]); }); }); + +describe("teamPerModelLimitOptions", () => { + const USER_MODELS = ["gpt-4o", "haiku", "openai/gpt-4o-mini", "openai/gpt-4.1"]; + + it("offers access-group models, not the sentinel, for a no-default-models team", () => { + expect(teamPerModelLimitOptions(["no-default-models"], ["gpt-4o", "haiku"], USER_MODELS)).toEqual([ + "gpt-4o", + "haiku", + ]); + }); + + it("merges direct and access-group models without duplicates", () => { + expect(teamPerModelLimitOptions(["gpt-4o"], ["gpt-4o", "haiku"], USER_MODELS)).toEqual(["gpt-4o", "haiku"]); + }); + + it("offers every user model when the team covers all models", () => { + expect(teamPerModelLimitOptions([], [], USER_MODELS)).toEqual(USER_MODELS); + expect(teamPerModelLimitOptions(["all-proxy-models"], [], USER_MODELS)).toEqual(USER_MODELS); + expect(teamPerModelLimitOptions(["all-team-models"], [], USER_MODELS)).toEqual(USER_MODELS); + }); + + it("expands wildcards to concrete names and drops the pattern itself", () => { + expect(teamPerModelLimitOptions(["no-default-models"], ["openai/*"], USER_MODELS)).toEqual([ + "openai/gpt-4o-mini", + "openai/gpt-4.1", + ]); + }); + + it("offers nothing when the team has no models it can call", () => { + expect(teamPerModelLimitOptions(["no-default-models"], [], USER_MODELS)).toEqual([]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/team/teamModelAccess.ts b/ui/litellm-dashboard/src/components/team/teamModelAccess.ts index d5d9245ed51..3027fbb8686 100644 --- a/ui/litellm-dashboard/src/components/team/teamModelAccess.ts +++ b/ui/litellm-dashboard/src/components/team/teamModelAccess.ts @@ -1,3 +1,5 @@ +import { unfurlWildcardModelsInList } from "../key_team_helpers/fetch_available_models_team_key"; + export const ALL_PROXY_MODELS = "all-proxy-models"; export const NO_DEFAULT_MODELS = "no-default-models"; @@ -18,6 +20,24 @@ export interface TeamModelBadge { tooltip: string; } +const ALL_TEAM_MODELS = "all-team-models"; +const MODEL_SENTINELS: ReadonlySet = new Set([ALL_PROXY_MODELS, ALL_TEAM_MODELS, NO_DEFAULT_MODELS]); + +// Per-model rate limits and budgets are enforced on the model name a request sends, +// so only concrete names are offered: sentinels and wildcard patterns never match one. +export function teamPerModelLimitOptions( + teamModels: string[], + accessGroupModels: string[], + userModels: string[], +): string[] { + const coversAllModels = + teamModels.length === 0 || teamModels.includes(ALL_PROXY_MODELS) || teamModels.includes(ALL_TEAM_MODELS); + const granted = coversAllModels ? userModels : [...new Set([...teamModels, ...accessGroupModels])]; + return unfurlWildcardModelsInList(granted, userModels).filter( + (model) => !MODEL_SENTINELS.has(model) && !model.endsWith("*"), + ); +} + export function normalizeTeamModelSelection(models: string[] | undefined): string[] { return models && models.length > 0 ? models : [NO_DEFAULT_MODELS]; } From 0ccae7ab353ee7f3184b084cd23b0970db487636 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Tue, 20 Oct 2026 15:06:24 -0700 Subject: [PATCH 005/235] refactor(rust-bridge): decide native routing in the catalog, drop the bypass predicates (#46750) * refactor(rust-bridge): decide native routing in the catalog, drop the bypass predicates * refactor(rust-bridge): simplify catalog compatibility checks --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-unit.yml | 7 + litellm/chat_completions/dispatch.py | 9 +- litellm/embeddings/dispatch.py | 18 +- litellm/messages/dispatch.py | 68 ++-- litellm/ocr/dispatch.py | 17 +- litellm/responses/dispatch.py | 9 +- litellm/rust_bridge/AGENTS.md | 6 + litellm/rust_bridge/catalog.py | 29 +- litellm/rust_bridge/dispatch.py | 27 +- litellm/rust_bridge/public_call.py | 26 ++ tests/test_litellm_rust/AGENTS.md | 16 +- .../chat_completions/test_dispatch.py | 166 +++------- .../embeddings/test_dispatch.py | 50 +-- .../messages/test_dispatch.py | 207 ++++-------- tests/test_litellm_rust/ocr/test_dispatch.py | 224 +++++++++++++ .../ocr/test_public_dispatch.py | 300 ------------------ .../responses/test_dispatch.py | 77 +---- tests/test_litellm_rust/support/routing.py | 8 +- tests/test_litellm_rust/test_dispatch.py | 115 +++---- .../test_litellm_rust/test_public_entries.py | 287 +++++++++++++++++ tests/unit/rust_bridge/AGENTS.md | 2 +- tests/unit/rust_bridge/test_catalog.py | 57 +--- tests/unit/rust_bridge/test_public_call.py | 16 +- 23 files changed, 857 insertions(+), 884 deletions(-) delete mode 100644 tests/test_litellm_rust/ocr/test_public_dispatch.py create mode 100644 tests/test_litellm_rust/test_public_entries.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index eefda48b682..2b9263f0430 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -433,6 +433,13 @@ jobs: - shard: rust-bridge-harness test-path: >- + tests/test_litellm_rust/test_dispatch.py + tests/test_litellm_rust/test_public_entries.py + tests/test_litellm_rust/chat_completions/test_dispatch.py + tests/test_litellm_rust/embeddings/test_dispatch.py + tests/test_litellm_rust/messages/test_dispatch.py + tests/test_litellm_rust/ocr/test_dispatch.py + tests/test_litellm_rust/responses/test_dispatch.py tests/unit/compression tests/unit/harness tests/unit/rust_bridge diff --git a/litellm/chat_completions/dispatch.py b/litellm/chat_completions/dispatch.py index 99900177820..cfdcb76a26d 100644 --- a/litellm/chat_completions/dispatch.py +++ b/litellm/chat_completions/dispatch.py @@ -10,6 +10,7 @@ from litellm.rust_bridge.chat_completions.entrypoints import ( ) from litellm.rust_bridge.dispatch import PublicDispatch from litellm.rust_bridge.public_call import ( + AsyncFlag, NativeCall, bind, native_call, @@ -44,6 +45,7 @@ def _python_acompletion() -> PythonAcompletion: _PYTHON_COMPLETION: Final = _python_completion() _COMPLETION: Final = signature(_PYTHON_COMPLETION) +_FLAG: Final = AsyncFlag("acompletion", sync=_COMPLETION) _PYTHON_ACOMPLETION: Final = _python_acompletion() _ACOMPLETION: Final = signature(_PYTHON_ACOMPLETION) @@ -73,7 +75,6 @@ _DISPATCH: Final = PublicDispatch( route=Route.CHAT_COMPLETIONS, request=lambda args, kwargs: _public_request(_COMPLETION, args, kwargs), context=_context, - bypass=lambda request: request.kwargs.get("acompletion") is True, ) _ADISPATCH: Final = PublicDispatch( @@ -88,9 +89,11 @@ def completion( **kwargs: object, # kwargs-ok: preserve the public chat completions call shape ) -> ChatResult | Coroutine[object, object, ChatResult]: python: Final = _PYTHON_COMPLETION + if _FLAG.requested(kwargs): + return acompletion(**_FLAG.keywords(args, kwargs)) return _DISPATCH.run( args, - kwargs, + _FLAG.dropped(kwargs), python=python, binding=NATIVE_COMPLETION, native=native_call_hook, @@ -101,7 +104,7 @@ async def acompletion(*args: object, **kwargs: object) -> ChatResult: # kwargs- python: Final = _PYTHON_ACOMPLETION return await _ADISPATCH.arun( args, - kwargs, + _FLAG.dropped(kwargs), python=python, binding=NATIVE_ACOMPLETION, native=native_call_hook, diff --git a/litellm/embeddings/dispatch.py b/litellm/embeddings/dispatch.py index 3fd8893464d..772ac68737d 100644 --- a/litellm/embeddings/dispatch.py +++ b/litellm/embeddings/dispatch.py @@ -9,7 +9,15 @@ from litellm.rust_bridge.embeddings.entrypoints import ( NATIVE_AEMBEDDING, NATIVE_EMBEDDING, ) -from litellm.rust_bridge.public_call import NativeCall, bind, native_call, native_call_hook, optional_str, signature +from litellm.rust_bridge.public_call import ( + AsyncFlag, + NativeCall, + bind, + native_call, + native_call_hook, + optional_str, + signature, +) from litellm.types.utils import EmbeddingResponse __all__ = ("aembedding", "embedding") @@ -24,6 +32,7 @@ _PYTHON_AEMBEDDING: Final = cast( # cast-ok: [LIT006] preserve the legacy publi PythonAembedding, main.aembedding ) _EMBEDDING_SIGNATURE: Final = signature(_PYTHON_EMBEDDING) +_FLAG: Final = AsyncFlag("aembedding", sync=_EMBEDDING_SIGNATURE) def _public_request( @@ -50,7 +59,6 @@ _DISPATCH: Final = PublicDispatch( route=Route.EMBEDDINGS, request=lambda args, kwargs: _public_request(_EMBEDDING_SIGNATURE, args, kwargs), context=_context, - bypass=lambda request: request.kwargs.get("aembedding") is True, ) _ADISPATCH: Final = PublicDispatch( @@ -64,9 +72,11 @@ def embedding( *args: object, **kwargs: object, # kwargs-ok: preserve the public embedding call shape ) -> EmbeddingResponse | Coroutine[object, object, EmbeddingResponse]: + if _FLAG.requested(kwargs): + return aembedding(**_FLAG.keywords(args, kwargs)) return _DISPATCH.run( args, - kwargs, + _FLAG.dropped(kwargs), python=_PYTHON_EMBEDDING, binding=NATIVE_EMBEDDING, native=native_call_hook, @@ -76,7 +86,7 @@ def embedding( async def aembedding(*args: object, **kwargs: object) -> EmbeddingResponse: # kwargs-ok: preserve the public call shape return await _ADISPATCH.arun( args, - kwargs, + _FLAG.dropped(kwargs), python=_PYTHON_AEMBEDDING, binding=NATIVE_AEMBEDDING, native=native_call_hook, diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index cb6ed8d5c75..4851d7e01b4 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -1,18 +1,20 @@ import inspect from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping +from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.anthropic.pass_through.messages import handler as main from litellm.llms.anthropic.pass_through.safeguards import native_route_classifies -from litellm.rust_bridge.catalog import Route, RouteContext +from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target from litellm.rust_bridge.dispatch import PublicDispatch from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, NATIVE_MESSAGES, ) from litellm.rust_bridge.public_call import ( + AsyncFlag, NativeCall, bind, native_call, @@ -46,6 +48,7 @@ def _python_amessages() -> PythonAmessages: _PYTHON_MESSAGES: Final = _python_messages() _MESSAGES: Final = signature(_PYTHON_MESSAGES) +_FLAG: Final = AsyncFlag("is_async", sync=_MESSAGES, is_set=bool) _PYTHON_AMESSAGES: Final = _python_amessages() _AMESSAGES: Final = signature(_PYTHON_AMESSAGES) @@ -71,30 +74,49 @@ def _public_request( return native_call(legacy, args, _native_kwargs(fields, kwargs)) -def _resolved_provider(request: NativeCall) -> str | None: - try: - return get_llm_provider( - str(request.resolved["model"]), optional_str(request.resolved.get("custom_llm_provider")) - )[1] - except BadRequestError: - return optional_str(request.resolved.get("custom_llm_provider")) - - -def _python_serves(request: NativeCall) -> bool: - """Vertex AI serves only Claude through the Anthropic Messages config, as - ``get_provider_anthropic_messages_config`` does; every other Vertex model keeps Python's - translation through Chat Completions. A request Python would answer with classifier - ``safeguard_results`` stays on Python too.""" +def _resolved(request: NativeCall) -> tuple[str, str | None]: model: Final = str(request.resolved["model"]) - vertex_without_claude: Final = _resolved_provider(request) == "vertex_ai" and "claude" not in model.lower() - return vertex_without_claude or native_route_classifies(model, request.kwargs.get("safeguards")) + provider: Final = optional_str(request.resolved.get("custom_llm_provider")) + try: + resolved_model, resolved_provider, _, _ = get_llm_provider(model, provider) + except BadRequestError: + return model, provider + return resolved_model, resolved_provider + + +def _target(request: NativeCall, model: str, provider: str | None) -> Target: + requested_model: Final = str(request.resolved["model"]) + model_info: Final = request.resolved.get("model_info") + api_base: Final = request.resolved.get("api_base") + native_config: Final = main._native_messages_config( # pyright: ignore[reportPrivateUsage] # the handler's own passthrough lookup + model, provider, model_info, api_base if isinstance(api_base, str) else None + ) + if native_config is not None: + return Target.MESSAGES + compacting: Final = request.resolved.get("compaction") is not None + if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice + return Target.RESPONSES + return Target.CHAT_COMPLETIONS + + +_FEATURES: Final[Mapping[Feature, Callable[[NativeCall], bool]]] = MappingProxyType( + { + Feature.SAFEGUARDS_CLASSIFIER: lambda request: native_route_classifies( + str(request.resolved["model"]), request.resolved.get("safeguards") + ), + } +) def _context(request: NativeCall) -> RouteContext: + requested_model: Final = str(request.resolved["model"]) + model, provider = _resolved(request) return RouteContext( Route.MESSAGES, - provider=_resolved_provider(request), - model=str(request.resolved["model"]), + provider=provider, + model=requested_model, + target=_target(request, model, provider), + features=frozenset(feature for feature, detected in _FEATURES.items() if detected(request)), ) @@ -102,14 +124,12 @@ _DISPATCH: Final = PublicDispatch( route=Route.MESSAGES, request=lambda args, kwargs: _public_request(_MESSAGES, args, kwargs), context=_context, - bypass=lambda request: request.kwargs.get("is_async") is True or _python_serves(request), ) _ADISPATCH: Final = PublicDispatch( route=Route.MESSAGES, request=lambda args, kwargs: _public_request(_AMESSAGES, args, kwargs), context=_context, - bypass=_python_serves, ) @@ -118,9 +138,11 @@ def anthropic_messages_handler( **kwargs: object, # kwargs-ok: preserve the public Anthropic Messages call shape ) -> MessagesResult | Coroutine[object, object, MessagesResult]: python: Final = _PYTHON_MESSAGES + if _FLAG.requested(kwargs): + return anthropic_messages(**_FLAG.keywords(args, kwargs)) return _DISPATCH.run( args, - kwargs, + _FLAG.dropped(kwargs), python=python, binding=NATIVE_MESSAGES, native=native_call_hook, @@ -131,7 +153,7 @@ async def anthropic_messages(*args: object, **kwargs: object) -> MessagesResult: python: Final = _PYTHON_AMESSAGES return await _ADISPATCH.arun( args, - kwargs, + _FLAG.dropped(kwargs), python=python, binding=NATIVE_AMESSAGES, native=native_call_hook, diff --git a/litellm/ocr/dispatch.py b/litellm/ocr/dispatch.py index 4fb4a6c709a..529c3df2bb9 100644 --- a/litellm/ocr/dispatch.py +++ b/litellm/ocr/dispatch.py @@ -9,7 +9,14 @@ from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR -from litellm.rust_bridge.public_call import NativeCall, native_call, native_call_hook, optional_str, signature +from litellm.rust_bridge.public_call import ( + AsyncFlag, + NativeCall, + native_call, + native_call_hook, + optional_str, + signature, +) __all__ = ("aocr", "ocr") @@ -39,6 +46,7 @@ def _bind_request( _OCR: Final = signature(_bind_request) +_FLAG: Final = AsyncFlag("aocr", sync=_OCR) def _public_request(name: str, args: tuple[object, ...], kwargs: Mapping[str, object]) -> NativeCall: @@ -59,7 +67,6 @@ _DISPATCH: Final = PublicDispatch( route=Route.OCR, request=lambda args, kwargs: _public_request("ocr", args, kwargs), context=_context, - bypass=lambda request: request.kwargs.get("aocr") is True, ) _ADISPATCH: Final = PublicDispatch( @@ -73,9 +80,11 @@ def ocr( *args: object, **kwargs: object, # kwargs-ok: preserve the public OCR call shape ) -> OCRResponse | Coroutine[object, object, OCRResponse]: + if _FLAG.requested(kwargs): + return aocr(**_FLAG.keywords(args, kwargs)) return _DISPATCH.run( args, - kwargs, + _FLAG.dropped(kwargs), python=runtime.NO_PYTHON, binding=NATIVE_OCR, native=native_call_hook, @@ -85,7 +94,7 @@ def ocr( async def aocr(*args: object, **kwargs: object) -> OCRResponse: # kwargs-ok: preserve the public OCR call shape return await _ADISPATCH.arun( args, - kwargs, + _FLAG.dropped(kwargs), python=runtime.NO_PYTHON, binding=NATIVE_AOCR, native=native_call_hook, diff --git a/litellm/responses/dispatch.py b/litellm/responses/dispatch.py index ad642fb427c..281aa177bae 100644 --- a/litellm/responses/dispatch.py +++ b/litellm/responses/dispatch.py @@ -7,6 +7,7 @@ from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterat from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch from litellm.rust_bridge.public_call import ( + AsyncFlag, NativeCall, bind, native_call, @@ -43,6 +44,7 @@ def _python_aresponses() -> PythonAresponses: _PYTHON_RESPONSES: Final = _python_responses() _RESPONSES: Final = signature(_PYTHON_RESPONSES) +_FLAG: Final = AsyncFlag("aresponses", sync=_RESPONSES) _PYTHON_ARESPONSES: Final = _python_aresponses() _ARESPONSES: Final = signature(_PYTHON_ARESPONSES) @@ -71,7 +73,6 @@ _DISPATCH: Final = PublicDispatch( route=Route.RESPONSES, request=lambda args, kwargs: _public_request(_RESPONSES, args, kwargs), context=_context, - bypass=lambda request: request.kwargs.get("aresponses") is True, ) _ADISPATCH: Final = PublicDispatch( @@ -86,9 +87,11 @@ def responses( **kwargs: object, # kwargs-ok: preserve the public Responses call shape ) -> ResponsesResult | Coroutine[object, object, ResponsesResult]: python: Final = _PYTHON_RESPONSES + if _FLAG.requested(kwargs): + return aresponses(**_FLAG.keywords(args, kwargs)) return _DISPATCH.run( args, - kwargs, + _FLAG.dropped(kwargs), python=python, binding=NATIVE_RESPONSES, native=native_call_hook, @@ -99,7 +102,7 @@ async def aresponses(*args: object, **kwargs: object) -> ResponsesResult: # kwa python: Final = _PYTHON_ARESPONSES return await _ADISPATCH.arun( args, - kwargs, + _FLAG.dropped(kwargs), python=python, binding=NATIVE_ARESPONSES, native=native_call_hook, diff --git a/litellm/rust_bridge/AGENTS.md b/litellm/rust_bridge/AGENTS.md index 607bb257a0e..6c072684f84 100644 --- a/litellm/rust_bridge/AGENTS.md +++ b/litellm/rust_bridge/AGENTS.md @@ -13,3 +13,9 @@ Await each selected suspension in the caller's task so hooks retain thread, loop Native admission may select legacy before execution starts. A failure after execution starts is terminal and never authorizes fallback. Creating an unstarted native coroutine must not acquire clients or credentials or capture execution context Verify cross-language behavior using a fresh installed extension, including unstarted-call release, setup ordering, exception identity, sync and async streams, cancellation and GC. Keep `_native.pyi` accurate about Future-returning and coroutine-returning APIs + +A route with no Python implementation passes `runtime.NO_PYTHON` from both public entries, and every catalog rule for it must be `RUST_REQUIRED`. Dispatch raises `NoPythonImplementationError` for any other rollout instead of falling back. `tests/test_litellm_rust/test_public_entries.py` marks such a route with `python=None`, so dropping a route's Python changes the entries, its rules and that one table field together + +A route's context, its target as well as its features, is built from `NativeCall.resolved`, so an argument counts the same whether it came by position, by keyword or as a signature default + +Every public route serves `a(...)`, `(..., a=True)` and `a(..., a=True)` the same way. Each hands native one identical `NativeCall` with no async flag, on the async binding. `test_public_entries.py` holds this for every route in its table, so a new public route gets a row there diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index ca271e201b0..37548943c80 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -25,11 +25,23 @@ class Route(str, Enum): TOKENIZER = "tokenizer" +class Target(str, Enum): + CHAT_COMPLETIONS = "chat_completions" + MESSAGES = "messages" + RESPONSES = "responses" + + +class Feature(str, Enum): + SAFEGUARDS_CLASSIFIER = "safeguards_classifier" + + @dataclass(frozen=True, slots=True) class RouteContext: route: Route provider: str | None = None model: str | None = None + target: Target | None = None + features: frozenset[Feature] = frozenset() @dataclass(frozen=True, slots=True) @@ -38,12 +50,19 @@ class RouteRule: rollout: Rollout providers: frozenset[str] | None = None models: frozenset[str] | None = None + targets: frozenset[Target] | None = None + features: frozenset[Feature] | None = None + + def unconstrained(self) -> bool: + return self.providers is None and self.models is None and self.targets is None and self.features is None def matches(self, context: RouteContext) -> bool: return ( context.route is self.route and (self.providers is None or context.provider in self.providers) and (self.models is None or context.model in self.models) + and (self.targets is None or context.target in self.targets) + and (self.features is None or not self.features.isdisjoint(context.features)) ) @@ -53,7 +72,9 @@ RULES: Final[Rules] = ( RouteRule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY), RouteRule(Route.EMBEDDINGS, Rollout.PYTHON_ONLY), RouteRule(Route.OCR, Rollout.RUST_REQUIRED), - RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN, providers=frozenset({"anthropic", "deepseek", "vertex_ai"})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, targets=frozenset({Target.CHAT_COMPLETIONS, Target.RESPONSES})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), + RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN, providers=frozenset({"anthropic"})), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY), @@ -69,3 +90,9 @@ def rollout(context: RouteContext, rules: Rules | None = None) -> Rollout: def decision(context: RouteContext, rules: Rules | None = None) -> Decision: return _decision(rollout(context, rules)) + + +def rust_might_handle(route: Route, rules: Rules) -> bool: + own: Final = tuple(rule for rule in rules if rule.route is route) + last: Final = next((i for i, rule in enumerate(own) if rule.unconstrained()), len(own)) + return any(_decision(rule.rollout) is not Decision.PYTHON for rule in own[: last + 1]) diff --git a/litellm/rust_bridge/dispatch.py b/litellm/rust_bridge/dispatch.py index da9b9bed54f..bfc449ca3bc 100644 --- a/litellm/rust_bridge/dispatch.py +++ b/litellm/rust_bridge/dispatch.py @@ -7,8 +7,6 @@ from typing import Final, Generic, TypeAlias, TypeVar from litellm.rust_bridge import catalog, runtime from litellm.rust_bridge.bindings import NativeBinding from litellm.rust_bridge.catalog import Route, RouteContext, Rules -from litellm.rust_bridge.configuration import Decision -from litellm.rust_bridge.configuration import decision as rollout_decision RequestT: Final = TypeVar("RequestT") NativeT: Final = TypeVar("NativeT") @@ -31,18 +29,6 @@ class PublicDispatch(Generic[RequestT]): route: Route request: Callable[[tuple[object, ...], Mapping[str, object]], RequestT | None] context: Callable[[RequestT], RouteContext] - bypass: Callable[[RequestT], bool] | None = None - - def _requires_projection(self, rules: Rules) -> bool: - for rule in rules: - if rule.route is not self.route: - continue - if rule.providers is not None or rule.models is not None: - if rollout_decision(rule.rollout) is not Decision.PYTHON: - return True - continue - return rollout_decision(rule.rollout) is not Decision.PYTHON - return False def _native_request(self, args: tuple[object, ...], kwargs: Mapping[str, object]) -> RequestT: request: Final = self.request(args, kwargs) @@ -50,11 +36,6 @@ class PublicDispatch(Generic[RequestT]): raise runtime.NoPythonImplementationError( f"{self.route.value} has no Python implementation, so every call must project to a native request" ) - if self.bypass is not None and self.bypass(request): - raise runtime.NoPythonImplementationError( - f"{self.route.value} has no Python implementation, so a call its bypass predicate matches cannot " - "be served" - ) return request def run( @@ -77,10 +58,10 @@ class PublicDispatch(Generic[RequestT]): python=python, rules=selected_rules, ) - if not self._requires_projection(selected_rules): + if not catalog.rust_might_handle(self.route, selected_rules): return python(*args, **kwargs) request: Final = self.request(args, kwargs) - if request is None or (self.bypass is not None and self.bypass(request)): + if request is None: return python(*args, **kwargs) return runtime.run( self.context(request), @@ -110,10 +91,10 @@ class PublicDispatch(Generic[RequestT]): python=python, rules=selected_rules, ) - if not self._requires_projection(selected_rules): + if not catalog.rust_might_handle(self.route, selected_rules): return await python(*args, **kwargs) request: Final = self.request(args, kwargs) - if request is None or (self.bypass is not None and self.bypass(request)): + if request is None: return await python(*args, **kwargs) return await runtime.arun( self.context(request), diff --git a/litellm/rust_bridge/public_call.py b/litellm/rust_bridge/public_call.py index 332421c68cc..4b69074d057 100644 --- a/litellm/rust_bridge/public_call.py +++ b/litellm/rust_bridge/public_call.py @@ -113,6 +113,32 @@ def _without_bags(legacy: inspect.Signature, named: Mapping[str, object]) -> Map return MappingProxyType({name: value for name, value in named.items() if legacy.parameters[name].kind not in _BAGS}) +@dataclass(frozen=True, slots=True) +class AsyncFlag: + """The keyword that makes a sync entry hand off to its async entry, as in ``completion(..., acompletion=True)``. + + ``is_set`` is the route's own reading of the flag. ``@client`` hands off on ``is True``, the Messages handler on truthiness. + + Completion uses ``AsyncFlag("acompletion", sync=signature(_PYTHON_COMPLETION))``. + Messages uses ``AsyncFlag("is_async", sync=signature(_PYTHON_MESSAGES), is_set=bool)``. + """ + + name: str + sync: inspect.Signature + is_set: Callable[[object], bool] = lambda value: value is True + + def requested(self, kwargs: Mapping[str, object]) -> bool: + return self.is_set(kwargs.get(self.name)) + + def dropped(self, kwargs: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType({name: value for name, value in kwargs.items() if name != self.name}) + + def keywords(self, args: tuple[object, ...], kwargs: Mapping[str, object]) -> Mapping[str, object]: + """Name positionals by the sync signature, since the async entry may order its parameters differently.""" + named: Final = _without_bags(self.sync, self.sync.bind_partial(*args, **kwargs).arguments) + return self.dropped({**named, **kwargs}) + + def native_call(legacy: inspect.Signature, args: tuple[object, ...], kwargs: Mapping[str, object]) -> NativeCall: positional: Final = legacy.bind_partial(*args) positional.apply_defaults() diff --git a/tests/test_litellm_rust/AGENTS.md b/tests/test_litellm_rust/AGENTS.md index ea661bc36f1..e115819be9d 100644 --- a/tests/test_litellm_rust/AGENTS.md +++ b/tests/test_litellm_rust/AGENTS.md @@ -1,20 +1,24 @@ # Invariants -- This directory tests the Rust path and nothing else. Treat the Python implementation of every route as already removed. Never call it on purpose, never run one request through both engines, and never compare Python and Rust outputs, headers, cache keys or logs -- A test that would still pass with Python serving the call does not belong here. Every test asserts the call was served natively with `support.routing.assert_served_natively` +- Outside the dispatch tests below, this directory tests the Rust path and nothing else. Treat the Python implementation of every route as already removed. Never call it on purpose, never run one request through both engines, and never compare Python and Rust outputs, headers, cache keys or logs +- A test that would still pass with Python serving the call does not belong here. Every test except the dispatch tests asserts the call was served natively with `support.routing.assert_served_natively` +- Dispatch tests check which path a public call takes under given rules. They are each route folder's `test_dispatch.py`, plus `test_dispatch.py` and `test_public_entries.py` at the top of this directory + - They pass `rules=` to a dispatch directly or wrap the whole test in one `support.routing.routed(...)` block, and they stub the native bindings + - They may call Python, since the choice between the two paths is what they test + - They live here until they can run in `tests/unit/` without the compiled extension - Select the Rust path with `support.routing.native_route(...)`, which prepends `RUST_REQUIRED` rules to `catalog.RULES` and restores them on exit. Each route folder's `conftest.py` applies it to every test through an autouse fixture. A test outside a route folder wraps its whole body in one `native_route(...)` block, and a child interpreter script assigns `catalog.RULES` once at its top. Never select a backend by reading or setting `LITELLM_RUST`, calling `litellm.rust(...)` or rebinding `catalog.RULES` any other way. The rollout in `litellm/rust_bridge/catalog.py` is a product decision, and a test here must pass no matter what it says, including for a route marked `PYTHON_ONLY` - `catalog.RULES` is static at runtime and `native_route(...)` swaps a module global, so open it once per test and never per call. Two overlapping blocks, such as one per task inside an `asyncio.gather`, restore out of order and leave the wrong rules behind for later tests -- A test whose subject is the rollout switch itself (env, `litellm.rust(...)`, catalog rules) is a unit test of `litellm/rust_bridge/configuration.py` or `catalog.py` and lives in `tests/unit/rust_bridge/` +- No test here depends on the rollout switch. Only no rule, `PYTHON_ONLY` and `RUST_REQUIRED` pick a path without it, so those are the only rollouts a test here uses, and Rust with Python fallback is not reachable from this directory. What `LITELLM_RUST` and `litellm.rust(...)` mean is tested in `tests/unit/rust_bridge/test_configuration.py`, mostly through `decide`, which takes both as plain values - A test lives here only when it cannot be written in Rust. That means it needs the Python public API, Python-owned state such as `litellm.cache` and callbacks, or the process boundary (fork, threads, event loops). Anything checkable from the Rust side goes in the crate's own tests - The wire is the local recording server. No test here reaches a real provider -- Never parametrize one test across routes. Each route folder owns its tests, its `server` fixture, its request data and its `call` +- Never parametrize one test across routes. Each route folder owns its tests, its `server` fixture, its request data and its `call`. The one exception is `test_public_entries.py`, whose subject is that every route serves its public call forms the same way - Call `_native` directly only for a case the public API cannot reach # Layout Each route folder (`ocr/`, `messages/`, `responses/`, `chat_completions/`) uses the same file names, one concern per file -- `test_dispatch.py` the call reaches the native route +- `test_dispatch.py` the call reaches the native route, and which path the rules pick for each public call - `test_requests.py` Python inputs that reach the wire - `test_failures.py` errors raised to the caller and rejection before provider IO - `test_settings.py` globals, env, named credentials and their precedence @@ -24,4 +28,6 @@ Each route folder (`ocr/`, `messages/`, `responses/`, `chat_completions/`) uses - `test_lifecycle.py` cancellation, concurrency, context and nested calls - `test_cache.py` the configured cache seen from this route +`embeddings/` holds only `test_dispatch.py` so far. At the top, `test_dispatch.py` tests `PublicDispatch` with stub routes and `test_public_entries.py` runs every route's public call forms through one table + Route data and the route's `call` live in `/data.py`, and `/conftest.py` owns the `server` fixture and the autouse `native` fixture. `support/` holds only route-neutral helpers diff --git a/tests/test_litellm_rust/chat_completions/test_dispatch.py b/tests/test_litellm_rust/chat_completions/test_dispatch.py index 4ed75ea5f95..01249383910 100644 --- a/tests/test_litellm_rust/chat_completions/test_dispatch.py +++ b/tests/test_litellm_rust/chat_completions/test_dispatch.py @@ -9,7 +9,6 @@ from litellm.chat_completions.dispatch import ( _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch _DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch ) -from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding from litellm.rust_bridge.catalog import Route, RouteRule, Rules from litellm.rust_bridge.chat_completions.entrypoints import ( @@ -147,29 +146,6 @@ def test_native_receives_bound_request_and_original_call_shape() -> None: assert call_kwargs["metadata"] is metadata -def test_internal_async_marker_bypasses_native() -> None: - response: Final = ModelResponse() - called: Final[list[bool]] = [] - - def python(*call_args: object, **call_kwargs: object) -> ModelResponse: # kwargs-ok: records public call shape - called.append(True) - return response - - def native(request: NativeCall) -> ModelResponse: - pytest.fail("acompletion's inner completion call must stay on Python") - - result: Final = _DISPATCH.run( - ("gpt-4o", MESSAGES), - {"acompletion": True}, - python=python, - binding=completion_binding(native), - native=native_call_hook, - rules=RUST_RULES, - ) - assert result is response - assert called == [True] - - @pytest.mark.parametrize( ("args", "kwargs"), ( @@ -202,56 +178,6 @@ def test_binding_errors_delegate_to_python(args: tuple[object, ...], kwargs: Map assert captured == [(args, kwargs)] -def test_public_completion_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: - captured: Final[list[NativeCall]] = [] - expected: Final = ModelResponse() - - def native(request: NativeCall) -> ModelResponse: - captured.append(request) - return expected - - NATIVE_COMPLETION.override(native) - monkeypatch.setattr(catalog, "RULES", RUST_RULES) - public_completion: Final = cast(Callable[..., ModelResponse], litellm.completion) - try: - result: Final = public_completion(model="gpt-4o", messages=MESSAGES) - finally: - NATIVE_COMPLETION.reset() - assert result is expected - assert [request.resolved["model"] for request in captured] == ["gpt-4o"] - - -@pytest.mark.asyncio -async def test_public_acompletion_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: - captured: Final[list[NativeCall]] = [] - expected: Final = ModelResponse() - - async def native(request: NativeCall) -> ModelResponse: - captured.append(request) - return expected - - NATIVE_ACOMPLETION.override(native) - monkeypatch.setattr(catalog, "RULES", RUST_RULES) - public_acompletion: Final = cast(Callable[..., Awaitable[ModelResponse]], litellm.acompletion) - try: - result: Final = await public_acompletion(model="gpt-4o", messages=MESSAGES) - finally: - NATIVE_ACOMPLETION.reset() - assert result is expected - assert [request.resolved["model"] for request in captured] == ["gpt-4o"] - - -@pytest.mark.asyncio -async def test_public_completion_calls_keep_the_python_result() -> None: - sync_response: Final = litellm.completion(model="openai/test-model", messages=MESSAGES, mock_response="ok") - async_response: Final = await litellm.acompletion(model="openai/test-model", messages=MESSAGES, mock_response="ok") - - assert isinstance(sync_response, ModelResponse) - assert isinstance(async_response, ModelResponse) - assert sync_response.choices[0].message.content == "ok" - assert async_response.choices[0].message.content == "ok" - - def test_sync_completion_request_projects_public_arguments() -> None: rules: Final[Rules] = (RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),) expected: Final = ModelResponse() @@ -277,64 +203,46 @@ def test_sync_completion_request_projects_public_arguments() -> None: assert response is expected -@pytest.mark.asyncio -async def test_async_completion_falls_back_after_native_declines() -> None: - from litellm.rust_bridge.bindings import native_exception_types - - native_types: Final = native_exception_types() - if native_types is None: - pytest.skip("native bridge is unavailable") - declined, _ = native_types - expected: Final = ModelResponse() - rules: Final[Rules] = (RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_OPT_OUT),) - - async def native(request: NativeCall) -> ModelResponse: - raise declined("unsupported") - - async def python(*args: object, **kwargs: object) -> ModelResponse: - return expected - - binding: Final[NativeBinding[NativeAcompletion]] = NativeBinding("acompletion", validate=lambda _: None) - binding.override(native) - response: Final = await dispatch._ADISPATCH.arun( # pyright: ignore[reportPrivateUsage] # test an explicit route decision - ("test-model", MESSAGES), - {}, - python=python, - binding=binding, - native=native_call_hook, - rules=rules, - ) - - assert response is expected - - -def test_internal_acompletion_marker_bypasses_native() -> None: - rules: Final[Rules] = (RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),) - expected: Final = ModelResponse() - - def python(*args: object, **kwargs: object) -> ModelResponse: - return expected - - def native(request: NativeCall) -> ModelResponse: - pytest.fail("acompletion's inner completion call must stay on Python") - - binding: Final[NativeBinding[NativeCompletion]] = NativeBinding("completion", validate=lambda _: None) - binding.override(native) - response: Final = dispatch._DISPATCH.run( # pyright: ignore[reportPrivateUsage] # test an explicit route decision - ("test-model", MESSAGES), - {"custom_llm_provider": "openai", "acompletion": True}, - python=python, - binding=binding, - native=native_call_hook, - rules=rules, - ) - - assert response is expected - - def test_positional_parameters_remain_available_to_native_projection() -> None: request: Final = _DISPATCH.request(("anthropic/test-model", MESSAGES, 12.0, 0.25), {}) assert request is not None assert request.resolved["timeout"] == 12.0 assert request.resolved["temperature"] == 0.25 assert request.resolved["messages"] is MESSAGES + + +@pytest.mark.asyncio +async def test_acompletion_marker_binds_positionals_by_the_sync_signature() -> None: + captured: Final[list[NativeCall]] = [] + + async def native(call: NativeCall) -> ModelResponse: + captured.append(call) + return ModelResponse() + + NATIVE_ACOMPLETION.override(native) + public_completion: Final = cast(Callable[..., Awaitable[ModelResponse]], litellm.completion) + try: + _ = await public_completion("gpt-4o", MESSAGES, 30.0, acompletion=True) + finally: + NATIVE_ACOMPLETION.reset() + assert [(request.resolved["timeout"], request.resolved["functions"]) for request in captured] == [(30.0, None)] + + +def test_acompletion_flag_hands_off_only_when_true() -> None: + expected: Final = ModelResponse() + + def native(call: NativeCall) -> ModelResponse: + return expected + + async def anative(call: NativeCall) -> ModelResponse: + pytest.fail("completion(..., acompletion=1) stays sync, since the Python @client only hands off on is True") + + NATIVE_COMPLETION.override(native) + NATIVE_ACOMPLETION.override(anative) + public_completion: Final = cast(Callable[..., object], litellm.completion) + try: + result: Final = public_completion("gpt-4o", MESSAGES, acompletion=1) + finally: + NATIVE_COMPLETION.reset() + NATIVE_ACOMPLETION.reset() + assert result is expected diff --git a/tests/test_litellm_rust/embeddings/test_dispatch.py b/tests/test_litellm_rust/embeddings/test_dispatch.py index 750c88cd235..fbebf1c64e4 100644 --- a/tests/test_litellm_rust/embeddings/test_dispatch.py +++ b/tests/test_litellm_rust/embeddings/test_dispatch.py @@ -1,12 +1,10 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable +from collections.abc import Callable from typing import Final import pytest -from pydantic import TypeAdapter -import litellm from litellm.embeddings import dispatch from litellm.rust_bridge.bindings import NativeBinding from litellm.rust_bridge.catalog import Route, RouteRule, Rules @@ -15,19 +13,6 @@ from litellm.rust_bridge.public_call import NativeCall, native_call_hook from litellm.types.utils import EmbeddingResponse -@pytest.mark.asyncio -async def test_public_embedding_calls_keep_the_python_result() -> None: - vector: Final = [0.1, 0.2] - - sync_response: Final = litellm.embedding(model="openai/test-model", input="hello", mock_response=vector) - async_response: Final = await litellm.aembedding(model="openai/test-model", input="hello", mock_response=vector) - - assert isinstance(sync_response, EmbeddingResponse) - rows: Final = TypeAdapter(list[dict[str, object]]) - assert rows.validate_python(sync_response.model_dump()["data"])[0]["embedding"] == vector - assert rows.validate_python(async_response.model_dump()["data"])[0]["embedding"] == vector - - def test_sync_embedding_request_projects_public_arguments() -> None: rules: Final[Rules] = (RouteRule(Route.EMBEDDINGS, Rollout.RUST_REQUIRED),) expected: Final = EmbeddingResponse(model="test-model", data=[]) @@ -52,36 +37,3 @@ def test_sync_embedding_request_projects_public_arguments() -> None: ) assert response is expected - - -@pytest.mark.asyncio -async def test_async_embedding_falls_back_after_native_declines() -> None: - from litellm.rust_bridge.bindings import native_exception_types - - native_types: Final = native_exception_types() - if native_types is None: - pytest.skip("native bridge is unavailable") - declined, _ = native_types - expected: Final = EmbeddingResponse(model="test-model", data=[]) - rules: Final[Rules] = (RouteRule(Route.EMBEDDINGS, Rollout.RUST_OPT_OUT),) - - async def native(request: NativeCall) -> EmbeddingResponse: - raise declined("unsupported") - - async def python(*args: object, **kwargs: object) -> EmbeddingResponse: - return expected - - binding: Final[NativeBinding[Callable[[NativeCall], Awaitable[EmbeddingResponse]]]] = NativeBinding( - "aembedding", validate=lambda _: None - ) - binding.override(native) - response: Final = await dispatch._ADISPATCH.arun( # pyright: ignore[reportPrivateUsage] # test an explicit route decision - ("test-model", "hello"), - {}, - python=python, - binding=binding, - native=native_call_hook, - rules=rules, - ) - - assert response is expected diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 2a32e31c250..6a82dd20c7e 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -3,7 +3,6 @@ from collections.abc import Awaitable, Callable, Mapping from typing import Final, cast # noqa: TID251 # narrows legacy callable signatures for inspect import pytest -from pydantic import TypeAdapter import litellm from litellm.llms.anthropic.pass_through.messages import handler as python_messages @@ -13,17 +12,14 @@ from litellm.messages.dispatch import ( _DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch ) from litellm.proxy import proxy_server -from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, RouteRule, Rules +from litellm.rust_bridge.catalog import Feature, Route, RouteRule, Rules, Target from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES, NativeAmessages, NativeMessages from litellm.rust_bridge.public_call import NativeCall from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse MESSAGES: Final = [{"role": "user", "content": "hi"}] -DANGEROUS_TOOL_USE: Final = [{"type": "dangerous_tool_use", "classifier_context": {"permission_mode": "auto"}}] -CLASSIFIER_SETTINGS: Final = {"safeguards_classifier_model": "gpt-oss-120b"} PYTHON_RULES: Final[Rules] = () RUST_RULES: Final[Rules] = (RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED),) @@ -194,81 +190,44 @@ def test_native_streams_are_bounded_by_stream_timeout(stream: bool, expected_tim assert captured[0].resolved["timeout"] == expected_timeout -def test_internal_async_marker_bypasses_native() -> None: - args: Final[tuple[object, ...]] = (16, MESSAGES, "claude-sonnet-4-5") - kwargs: Final[Mapping[str, object]] = {"is_async": True} - captured: Final[list[tuple[tuple[object, ...], Mapping[str, object]]]] = [] - expected: Final = response() - - def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: records call shape - captured.append((call_args, call_kwargs)) - return expected - - def native( - request: NativeCall, - ) -> AnthropicMessagesResponse: - pytest.fail("The async handler's inner sync call must stay on Python") - - result: Final = _DISPATCH.run( - args, - kwargs, - python=python, - binding=messages_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - assert result is expected - assert captured == [(args, kwargs)] - - @pytest.mark.parametrize( - ("model", "custom_llm_provider"), - [ - ("vertex_ai/gemini-2.5-pro", None), - ("gemini-2.5-pro", "vertex_ai"), - ], + ("model", "kwargs", "passes_through"), + ( + ("vertex_ai/claude-sonnet-4-5@20250929", {}, True), + ("claude-sonnet-4-5@20250929", {"custom_llm_provider": "vertex_ai"}, True), + ("deepseek/deepseek-v4-flash", {}, True), + ("openai/gpt-5", {"model_info": {"supported_endpoints": ["/v1/messages"]}}, True), + ("vertex_ai/gemini-2.5-pro", {}, False), + ("gemini-2.5-pro", {"custom_llm_provider": "vertex_ai"}, False), + ("openai/gpt-5", {}, False), + ("groq/llama-3.3-70b-versatile", {}, False), + ("unknown-provider/unknown-model", {}, False), + ), ) -def test_vertex_models_other_than_claude_stay_on_python(model: str, custom_llm_provider: str | None) -> None: - args: Final[tuple[object, ...]] = (16, MESSAGES, model) - kwargs: Final[Mapping[str, object]] = {"custom_llm_provider": custom_llm_provider} - expected: Final = response(model) +def test_only_requests_that_pass_through_to_a_messages_api_reach_a_messages_target_rule( + model: str, kwargs: Mapping[str, object], passes_through: bool +) -> None: + python_result: Final = response(model) + native_result: Final = response(model) - def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: records call shape - return expected - - def native(request: NativeCall) -> AnthropicMessagesResponse: - pytest.fail("Rust serves only Claude on Vertex AI") + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: fallback + return python_result result: Final = _DISPATCH.run( - args, + (16, MESSAGES, model), kwargs, python=python, - binding=messages_binding(native), + binding=messages_binding(lambda request: native_result), native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, + rules=(RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED, targets=frozenset({Target.MESSAGES})),), ) - assert result is expected + assert result is (native_result if passes_through else python_result) -def test_vertex_claude_reaches_native() -> None: - args: Final[tuple[object, ...]] = (16, MESSAGES, "vertex_ai/claude-sonnet-4-5@20250929") - expected: Final = response("vertex_ai/claude-sonnet-4-5@20250929") - - def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: records call shape - pytest.fail("Claude on Vertex AI is served natively") - - result: Final = _DISPATCH.run( - args, - {}, - python=python, - binding=messages_binding(lambda request: expected), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - assert result is expected +DANGEROUS_TOOL_USE: Final = [{"type": "dangerous_tool_use", "classifier_context": {"permission_mode": "auto"}}] +CLASSIFIER_SETTINGS: Final = {"safeguards_classifier_model": "gpt-oss-120b"} -@pytest.mark.asyncio @pytest.mark.parametrize( ("model", "kwargs", "general_settings", "serves_python"), ( @@ -285,7 +244,7 @@ def test_vertex_claude_reaches_native() -> None: ("anthropic/claude-sonnet-4-5", {"safeguards": DANGEROUS_TOOL_USE}, CLASSIFIER_SETTINGS, False), ), ) -async def test_python_keeps_the_requests_it_answers_with_safeguard_results( +def test_proxy_classified_safeguards_reach_the_feature_rule( monkeypatch: pytest.MonkeyPatch, model: str, kwargs: Mapping[str, object], @@ -296,19 +255,19 @@ async def test_python_keeps_the_requests_it_answers_with_safeguard_results( python_result: Final = response(model) native_result: Final = response(model) - async def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: fallback + def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: fallback return python_result - async def native(request: NativeCall) -> AnthropicMessagesResponse: - return native_result - - result: Final = await _ADISPATCH.arun( + result: Final = _DISPATCH.run( (16, MESSAGES, model), kwargs, python=python, - binding=amessages_binding(native), + binding=messages_binding(lambda request: native_result), native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, + rules=( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ), ) assert result is (python_result if serves_python else native_result) @@ -345,7 +304,7 @@ def test_binding_errors_delegate_to_python(args: tuple[object, ...], kwargs: Map assert captured == [(args, kwargs)] -def test_anthropic_create_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: +def test_anthropic_create_routes_through_dispatch() -> None: captured: Final[list[NativeCall]] = [] expected: Final = response() @@ -356,7 +315,6 @@ def test_anthropic_create_routes_through_dispatch(monkeypatch: pytest.MonkeyPatc return expected NATIVE_MESSAGES.override(native) - monkeypatch.setattr(catalog, "RULES", RUST_RULES) public_create: Final = cast(Callable[..., AnthropicMessagesResponse], litellm.anthropic.create) try: result: Final = public_create(max_tokens=16, messages=MESSAGES, model="claude-sonnet-4-5") @@ -367,7 +325,7 @@ def test_anthropic_create_routes_through_dispatch(monkeypatch: pytest.MonkeyPatc @pytest.mark.asyncio -async def test_anthropic_acreate_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_anthropic_acreate_routes_through_dispatch() -> None: captured: Final[list[NativeCall]] = [] expected: Final = response() @@ -378,7 +336,6 @@ async def test_anthropic_acreate_routes_through_dispatch(monkeypatch: pytest.Mon return expected NATIVE_AMESSAGES.override(native) - monkeypatch.setattr(catalog, "RULES", RUST_RULES) public_acreate: Final = cast(Callable[..., Awaitable[AnthropicMessagesResponse]], litellm.anthropic.acreate) try: result: Final = await public_acreate(max_tokens=16, messages=MESSAGES, model="claude-sonnet-4-5") @@ -389,14 +346,31 @@ async def test_anthropic_acreate_routes_through_dispatch(monkeypatch: pytest.Mon @pytest.mark.asyncio -async def test_public_anthropic_messages_keeps_the_python_result() -> None: - response: Final = await litellm.anthropic_messages( - model="anthropic/claude-sonnet-4-5", messages=MESSAGES, max_tokens=10, mock_response="ok" - ) +async def test_truthy_is_async_hands_off_to_the_async_entry_like_true() -> None: + captured: Final[list[NativeCall]] = [] + expected: Final = response() - assert isinstance(response, dict) - content: Final = TypeAdapter(list[dict[str, object]]).validate_python(response.get("content", [])) - assert content[0]["text"] == "ok" + async def native(request: NativeCall) -> AnthropicMessagesResponse: + captured.append(request) + return expected + + def sync_native(request: NativeCall) -> AnthropicMessagesResponse: + pytest.fail("a truthy is_async must hand off to the async entry, as the Python handler does") + + NATIVE_AMESSAGES.override(native) + NATIVE_MESSAGES.override(sync_native) + public_create: Final = cast(Callable[..., object], litellm.anthropic.create) + try: + pending: Final = public_create(max_tokens=16, messages=MESSAGES, model="claude-sonnet-4-5", is_async=1) + assert inspect.isawaitable(pending) + assert await pending is expected + reference: Final = public_create(max_tokens=16, messages=MESSAGES, model="claude-sonnet-4-5", is_async=True) + assert inspect.isawaitable(reference) + assert await reference is expected + finally: + NATIVE_AMESSAGES.reset() + NATIVE_MESSAGES.reset() + assert [dict(request.resolved) for request in captured] == [dict(captured[1].resolved)] * 2 def test_sync_messages_request_projects_public_arguments() -> None: @@ -451,64 +425,3 @@ def test_messages_binding_error_delegates_unchanged_to_python() -> None: ) assert response is expected - - -@pytest.mark.asyncio -async def test_async_messages_falls_back_after_native_declines() -> None: - from litellm.rust_bridge.bindings import native_exception_types - - native_types: Final = native_exception_types() - if native_types is None: - pytest.skip("native bridge is unavailable") - declined, _ = native_types - expected: Final = AnthropicMessagesResponse(model="claude-test") - rules: Final[Rules] = (RouteRule(Route.MESSAGES, Rollout.RUST_OPT_OUT),) - - async def native(request: NativeCall) -> AnthropicMessagesResponse: - raise declined("unsupported") - - async def python(*args: object, **kwargs: object) -> AnthropicMessagesResponse: - return expected - - binding: Final[NativeBinding[NativeAmessages]] = NativeBinding("amessages", validate=lambda _: None) - binding.override(native) - response: Final = await dispatch._ADISPATCH.arun( # pyright: ignore[reportPrivateUsage] # test an explicit route decision - (), - {"model": "claude-test", "messages": MESSAGES, "max_tokens": 10}, - python=python, - binding=binding, - native=lambda hook, request, args, kwargs: hook(request), - rules=rules, - ) - - assert response is expected - - -def test_internal_is_async_marker_bypasses_native() -> None: - rules: Final[Rules] = (RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED),) - expected: Final = AnthropicMessagesResponse(model="claude-test") - - def python(*args: object, **kwargs: object) -> AnthropicMessagesResponse: - return expected - - def native(request: NativeCall) -> AnthropicMessagesResponse: - pytest.fail("anthropic_messages' inner handler call must stay on Python") - - binding: Final[NativeBinding[NativeMessages]] = NativeBinding("messages", validate=lambda _: None) - binding.override(native) - response: Final = dispatch._DISPATCH.run( # pyright: ignore[reportPrivateUsage] # test an explicit route decision - (), - { - "model": "claude-test", - "messages": MESSAGES, - "max_tokens": 10, - "custom_llm_provider": "anthropic", - "is_async": True, - }, - python=python, - binding=binding, - native=lambda hook, request, args, kwargs: hook(request), - rules=rules, - ) - - assert response is expected diff --git a/tests/test_litellm_rust/ocr/test_dispatch.py b/tests/test_litellm_rust/ocr/test_dispatch.py index db60b6aea84..3206cc606a8 100644 --- a/tests/test_litellm_rust/ocr/test_dispatch.py +++ b/tests/test_litellm_rust/ocr/test_dispatch.py @@ -1,8 +1,21 @@ +from collections.abc import Mapping from typing import Final, NoReturn +import httpx import pytest import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.ocr.dispatch import ( + _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch + _DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch +) +from litellm.rust_bridge import runtime +from litellm.rust_bridge.bindings import NativeBinding +from litellm.rust_bridge.catalog import Route, RouteRule, Rules +from litellm.rust_bridge.configuration import Rollout +from litellm.rust_bridge.ocr.entrypoints import NativeAocr, NativeOcr +from litellm.rust_bridge.public_call import NativeCall from tests.test_litellm_rust.ocr.data import OCR_DOCUMENT, OCR_MODEL from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec from tests.test_litellm_rust.support.routing import assert_served_natively @@ -110,3 +123,214 @@ async def test_ocr_health_check_uses_the_native_route(recording_server: Recordin assert "error" not in response assert_served_natively(recording_server) assert recording_server.requests[0].body["document"]["image_url"].startswith("data:image/png;base64,") + + +RUST_RULES: Final[Rules] = (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),) + + +def ocr_binding(native: NativeOcr | None) -> NativeBinding[NativeOcr]: + binding: Final[NativeBinding[NativeOcr]] = NativeBinding("ocr", validate=lambda _: None) + binding.override(native) + return binding + + +def aocr_binding(native: NativeAocr | None) -> NativeBinding[NativeAocr]: + binding: Final[NativeBinding[NativeAocr]] = NativeBinding("aocr", validate=lambda _: None) + binding.override(native) + return binding + + +def response(model: str = "mistral/mistral-ocr-latest") -> OCRResponse: + return OCRResponse(pages=[], model=model) + + +def test_native_receives_normalized_positional_request_and_original_call_shape() -> None: + document: Final[Mapping[str, object]] = {"type": "file", "file": b"pdf"} + timeout: Final = httpx.Timeout(30) + extra_headers: Final[dict[str, object]] = {"x-test": "1"} + pages: Final = [0, 2] + args: Final[tuple[object, ...]] = ("mistral/mistral-ocr-latest", document) + kwargs: Final[Mapping[str, object]] = { + "api_key": "test-key", + "api_base": "https://example.invalid", + "timeout": timeout, + "custom_llm_provider": "mistral", + "extra_headers": extra_headers, + "pages": pages, + } + captured: Final[list[tuple[NativeCall, tuple[object, ...], Mapping[str, object]]]] = [] + expected: Final = response() + + def native( + request: NativeCall, + ) -> OCRResponse: + args: Final = request.args + kwargs: Final = request.kwargs + captured.append((request, args, kwargs)) + return expected + + result: Final = _DISPATCH.run( + args, + kwargs, + python=runtime.NO_PYTHON, + binding=ocr_binding(native), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=RUST_RULES, + ) + + request, call_args, call_kwargs = captured[0] + assert result is expected + assert request.resolved["model"] == "mistral/mistral-ocr-latest" + assert request.resolved["document"] is document + assert request.resolved["api_key"] == "test-key" + assert request.resolved["api_base"] == "https://example.invalid" + assert request.resolved["timeout"] is timeout + assert request.resolved["custom_llm_provider"] == "mistral" + assert request.resolved["extra_headers"] is extra_headers + assert request.kwargs == kwargs + assert request.kwargs["pages"] is pages + assert call_args is args + assert call_kwargs is kwargs + + +def test_native_preserves_keyword_model_and_document_in_original_call_shape() -> None: + document: Final[Mapping[str, object]] = { + "type": "document_url", + "document_url": "https://example.invalid/document.pdf", + } + pages: Final = [1] + args: Final[tuple[object, ...]] = () + kwargs: Final[Mapping[str, object]] = { + "model": "mistral/mistral-ocr-latest", + "document": document, + "pages": pages, + } + captured: Final[list[tuple[NativeCall, tuple[object, ...], Mapping[str, object]]]] = [] + expected: Final = response() + + def native( + request: NativeCall, + ) -> OCRResponse: + args: Final = request.args + kwargs: Final = request.kwargs + captured.append((request, args, kwargs)) + return expected + + result: Final = _DISPATCH.run( + args, + kwargs, + python=runtime.NO_PYTHON, + binding=ocr_binding(native), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=RUST_RULES, + ) + + request, call_args, call_kwargs = captured[0] + assert result is expected + assert request.resolved["model"] == "mistral/mistral-ocr-latest" + assert request.resolved["document"] is document + assert request.kwargs == kwargs + assert call_args is args + assert call_kwargs is kwargs + assert call_kwargs["model"] == "mistral/mistral-ocr-latest" + assert call_kwargs["document"] is document + + +def test_missing_native_binding_is_a_required_rust_error() -> None: + args: Final[tuple[object, ...]] = ("mistral/mistral-ocr-latest", {"type": "file", "file": b"pdf"}) + + with pytest.raises(RuntimeError, match="Rust ocr bridge"): + _DISPATCH.run( + args, + {}, + python=runtime.NO_PYTHON, + binding=ocr_binding(None), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=RUST_RULES, + ) + + +def test_non_required_rule_cannot_be_served_without_python() -> None: + args: Final[tuple[object, ...]] = ("mistral/mistral-ocr-latest", {"type": "file", "file": b"pdf"}) + + def native( + request: NativeCall, + ) -> OCRResponse: + return response() + + with pytest.raises(runtime.NoPythonImplementationError, match="RUST_REQUIRED"): + _DISPATCH.run( + args, + {}, + python=runtime.NO_PYTHON, + binding=ocr_binding(native), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=(RouteRule(Route.OCR, Rollout.PYTHON_ONLY),), + ) + + +@pytest.mark.parametrize( + ("args", "kwargs", "message"), + ( + ( + ("mistral/mistral-ocr-latest", {"type": "file", "file": b"pdf"}), + {"model": "duplicate"}, + r"ocr\(\) got multiple values for argument 'model'", + ), + ( + ("mistral/mistral-ocr-latest",), + {}, + r"ocr\(\) missing 1 required positional argument: 'document'", + ), + ), +) +def test_ocr_parser_errors_before_native(args: tuple[object, ...], kwargs: Mapping[str, object], message: str) -> None: + def native( + request: NativeCall, + ) -> OCRResponse: + pytest.fail("OCR parser failures must not call native") + + with pytest.raises(TypeError, match=message): + _DISPATCH.run( + args, + kwargs, + python=runtime.NO_PYTHON, + binding=ocr_binding(native), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=RUST_RULES, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("args", "kwargs", "message"), + ( + ( + ("mistral/mistral-ocr-latest", {"type": "file", "file": b"pdf"}), + {"model": "duplicate"}, + r"aocr\(\) got multiple values for argument 'model'", + ), + ( + ("mistral/mistral-ocr-latest",), + {}, + r"aocr\(\) missing 1 required positional argument: 'document'", + ), + ), +) +async def test_aocr_parser_errors_before_native( + args: tuple[object, ...], kwargs: Mapping[str, object], message: str +) -> None: + async def native( + request: NativeCall, + ) -> OCRResponse: + pytest.fail("OCR parser failures must not call native") + + with pytest.raises(TypeError, match=message): + await _ADISPATCH.arun( + args, + kwargs, + python=runtime.NO_PYTHON, + binding=aocr_binding(native), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=RUST_RULES, + ) diff --git a/tests/test_litellm_rust/ocr/test_public_dispatch.py b/tests/test_litellm_rust/ocr/test_public_dispatch.py deleted file mode 100644 index 34ee8e40552..00000000000 --- a/tests/test_litellm_rust/ocr/test_public_dispatch.py +++ /dev/null @@ -1,300 +0,0 @@ -from collections.abc import Awaitable, Callable, Mapping -from typing import Final, cast # noqa: TID251 # narrows legacy callable signatures for inspect - -import httpx -import pytest - -import litellm -from litellm.llms.base_llm.ocr.transformation import OCRResponse -from litellm.ocr.dispatch import ( - _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch - _DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch -) -from litellm.rust_bridge import catalog, runtime -from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, RouteRule, Rules -from litellm.rust_bridge.configuration import Rollout -from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR, NativeAocr, NativeOcr -from litellm.rust_bridge.public_call import NativeCall - -RUST_RULES: Final[Rules] = (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),) - - -def ocr_binding(native: NativeOcr | None) -> NativeBinding[NativeOcr]: - binding: Final[NativeBinding[NativeOcr]] = NativeBinding("ocr", validate=lambda _: None) - binding.override(native) - return binding - - -def aocr_binding(native: NativeAocr | None) -> NativeBinding[NativeAocr]: - binding: Final[NativeBinding[NativeAocr]] = NativeBinding("aocr", validate=lambda _: None) - binding.override(native) - return binding - - -def response(model: str = "mistral/mistral-ocr-latest") -> OCRResponse: - return OCRResponse(pages=[], model=model) - - -def test_native_receives_normalized_positional_request_and_original_call_shape() -> None: - document: Final[Mapping[str, object]] = {"type": "file", "file": b"pdf"} - timeout: Final = httpx.Timeout(30) - extra_headers: Final[dict[str, object]] = {"x-test": "1"} - pages: Final = [0, 2] - args: Final[tuple[object, ...]] = ("mistral/mistral-ocr-latest", document) - kwargs: Final[Mapping[str, object]] = { - "api_key": "test-key", - "api_base": "https://example.invalid", - "timeout": timeout, - "custom_llm_provider": "mistral", - "extra_headers": extra_headers, - "pages": pages, - } - captured: Final[list[tuple[NativeCall, tuple[object, ...], Mapping[str, object]]]] = [] - expected: Final = response() - - def native( - request: NativeCall, - ) -> OCRResponse: - args: Final = request.args - kwargs: Final = request.kwargs - captured.append((request, args, kwargs)) - return expected - - result: Final = _DISPATCH.run( - args, - kwargs, - python=runtime.NO_PYTHON, - binding=ocr_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - - request, call_args, call_kwargs = captured[0] - assert result is expected - assert request.resolved["model"] == "mistral/mistral-ocr-latest" - assert request.resolved["document"] is document - assert request.resolved["api_key"] == "test-key" - assert request.resolved["api_base"] == "https://example.invalid" - assert request.resolved["timeout"] is timeout - assert request.resolved["custom_llm_provider"] == "mistral" - assert request.resolved["extra_headers"] is extra_headers - assert request.kwargs == kwargs - assert request.kwargs["pages"] is pages - assert call_args is args - assert call_kwargs is kwargs - - -def test_native_preserves_keyword_model_and_document_in_original_call_shape() -> None: - document: Final[Mapping[str, object]] = { - "type": "document_url", - "document_url": "https://example.invalid/document.pdf", - } - pages: Final = [1] - args: Final[tuple[object, ...]] = () - kwargs: Final[Mapping[str, object]] = { - "model": "mistral/mistral-ocr-latest", - "document": document, - "pages": pages, - } - captured: Final[list[tuple[NativeCall, tuple[object, ...], Mapping[str, object]]]] = [] - expected: Final = response() - - def native( - request: NativeCall, - ) -> OCRResponse: - args: Final = request.args - kwargs: Final = request.kwargs - captured.append((request, args, kwargs)) - return expected - - result: Final = _DISPATCH.run( - args, - kwargs, - python=runtime.NO_PYTHON, - binding=ocr_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - - request, call_args, call_kwargs = captured[0] - assert result is expected - assert request.resolved["model"] == "mistral/mistral-ocr-latest" - assert request.resolved["document"] is document - assert request.kwargs == kwargs - assert call_args is args - assert call_kwargs is kwargs - assert call_kwargs["model"] == "mistral/mistral-ocr-latest" - assert call_kwargs["document"] is document - - -def test_aocr_marker_cannot_be_served_without_python() -> None: - document: Final[Mapping[str, object]] = {"type": "file", "file": b"pdf"} - args: Final[tuple[object, ...]] = ("mistral/mistral-ocr-latest", document) - kwargs: Final[Mapping[str, object]] = {"aocr": True} - - def native( - request: NativeCall, - ) -> OCRResponse: - pytest.fail("the aocr bypass marker must not reach native") - - with pytest.raises(runtime.NoPythonImplementationError, match="bypass"): - _DISPATCH.run( - args, - kwargs, - python=runtime.NO_PYTHON, - binding=ocr_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - - -def test_missing_native_binding_is_a_required_rust_error() -> None: - args: Final[tuple[object, ...]] = ("mistral/mistral-ocr-latest", {"type": "file", "file": b"pdf"}) - - with pytest.raises(RuntimeError, match="Rust ocr bridge"): - _DISPATCH.run( - args, - {}, - python=runtime.NO_PYTHON, - binding=ocr_binding(None), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - - -def test_non_required_rule_cannot_be_served_without_python() -> None: - args: Final[tuple[object, ...]] = ("mistral/mistral-ocr-latest", {"type": "file", "file": b"pdf"}) - - def native( - request: NativeCall, - ) -> OCRResponse: - return response() - - with pytest.raises(runtime.NoPythonImplementationError, match="RUST_REQUIRED"): - _DISPATCH.run( - args, - {}, - python=runtime.NO_PYTHON, - binding=ocr_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=(RouteRule(Route.OCR, Rollout.PYTHON_ONLY),), - ) - - -@pytest.mark.parametrize( - ("args", "kwargs", "message"), - ( - ( - ("mistral/mistral-ocr-latest", {"type": "file", "file": b"pdf"}), - {"model": "duplicate"}, - r"ocr\(\) got multiple values for argument 'model'", - ), - ( - ("mistral/mistral-ocr-latest",), - {}, - r"ocr\(\) missing 1 required positional argument: 'document'", - ), - ), -) -def test_ocr_parser_errors_before_native(args: tuple[object, ...], kwargs: Mapping[str, object], message: str) -> None: - def native( - request: NativeCall, - ) -> OCRResponse: - pytest.fail("OCR parser failures must not call native") - - with pytest.raises(TypeError, match=message): - _DISPATCH.run( - args, - kwargs, - python=runtime.NO_PYTHON, - binding=ocr_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("args", "kwargs", "message"), - ( - ( - ("mistral/mistral-ocr-latest", {"type": "file", "file": b"pdf"}), - {"model": "duplicate"}, - r"aocr\(\) got multiple values for argument 'model'", - ), - ( - ("mistral/mistral-ocr-latest",), - {}, - r"aocr\(\) missing 1 required positional argument: 'document'", - ), - ), -) -async def test_aocr_parser_errors_before_native( - args: tuple[object, ...], kwargs: Mapping[str, object], message: str -) -> None: - async def native( - request: NativeCall, - ) -> OCRResponse: - pytest.fail("OCR parser failures must not call native") - - with pytest.raises(TypeError, match=message): - await _ADISPATCH.arun( - args, - kwargs, - python=runtime.NO_PYTHON, - binding=aocr_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - - -def test_public_ocr_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: - document: Final[Mapping[str, object]] = { - "type": "document_url", - "document_url": "https://example.invalid/document.pdf", - } - captured: Final[list[NativeCall]] = [] - expected: Final = response() - - def native( - request: NativeCall, - ) -> OCRResponse: - captured.append(request) - return expected - - NATIVE_OCR.override(native) - monkeypatch.setattr(catalog, "RULES", RUST_RULES) - public_ocr: Final = cast(Callable[..., OCRResponse], litellm.ocr) - try: - result: Final = public_ocr(model="mistral/mistral-ocr-latest", document=document) - finally: - NATIVE_OCR.reset() - assert result is expected - assert [request.resolved["model"] for request in captured] == ["mistral/mistral-ocr-latest"] - - -@pytest.mark.asyncio -async def test_public_aocr_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: - document: Final[Mapping[str, object]] = { - "type": "document_url", - "document_url": "https://example.invalid/document.pdf", - } - captured: Final[list[NativeCall]] = [] - expected: Final = response() - - async def native( - request: NativeCall, - ) -> OCRResponse: - captured.append(request) - return expected - - NATIVE_AOCR.override(native) - monkeypatch.setattr(catalog, "RULES", RUST_RULES) - public_aocr: Final = cast(Callable[..., Awaitable[OCRResponse]], litellm.aocr) - try: - result: Final = await public_aocr(model="mistral/mistral-ocr-latest", document=document) - finally: - NATIVE_AOCR.reset() - assert result is expected - assert [request.resolved["model"] for request in captured] == ["mistral/mistral-ocr-latest"] diff --git a/tests/test_litellm_rust/responses/test_dispatch.py b/tests/test_litellm_rust/responses/test_dispatch.py index e6a0193a063..69535a9e6f3 100644 --- a/tests/test_litellm_rust/responses/test_dispatch.py +++ b/tests/test_litellm_rust/responses/test_dispatch.py @@ -1,5 +1,5 @@ import inspect -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Callable, Mapping from typing import Final, cast # noqa: TID251 # narrows legacy callable signatures for inspect import pytest @@ -11,14 +11,11 @@ from litellm.responses.dispatch import ( _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch _DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch ) -from litellm.rust_bridge import catalog from litellm.rust_bridge.bindings import NativeBinding from litellm.rust_bridge.catalog import Route, RouteRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.public_call import NativeCall from litellm.rust_bridge.responses.entrypoints import ( - NATIVE_ARESPONSES, - NATIVE_RESPONSES, NativeAresponses, NativeResponses, ) @@ -181,35 +178,6 @@ def test_native_receives_normalized_request_and_original_call_shape() -> None: assert call_kwargs["litellm_metadata"] is metadata -def test_internal_async_marker_bypasses_native() -> None: - args: Final[tuple[object, ...]] = (INPUT, "gpt-4o") - kwargs: Final[Mapping[str, object]] = {"aresponses": True} - captured: Final[list[tuple[tuple[object, ...], Mapping[str, object]]]] = [] - response: Final = _response() - - def python(*call_args: object, **call_kwargs: object) -> ResponsesAPIResponse: # kwargs-ok: records call shape - captured.append((call_args, call_kwargs)) - return response - - def native( - request: NativeCall, - ) -> ResponsesAPIResponse: - pytest.fail("aresponses' inner responses call must stay on Python") - - assert ( - _DISPATCH.run( - args, - kwargs, - python=python, - binding=responses_binding(native), - native=lambda hook, request, call_args, call_kwargs: hook(request), - rules=RUST_RULES, - ) - is response - ) - assert captured == [(args, kwargs)] - - @pytest.mark.parametrize( ("args", "kwargs"), ( @@ -244,49 +212,6 @@ def test_binding_errors_delegate_unchanged_to_python(args: tuple[object, ...], k assert captured == [(args, kwargs)] -def test_public_responses_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: - captured: Final[list[NativeCall]] = [] - expected: Final = _response() - - def native( - request: NativeCall, - ) -> ResponsesAPIResponse: - captured.append(request) - return expected - - NATIVE_RESPONSES.override(native) - monkeypatch.setattr(catalog, "RULES", RUST_RULES) - public_responses: Final = cast(Callable[..., ResponsesAPIResponse], litellm.responses) - try: - result: Final = public_responses(input=INPUT, model="gpt-4o") - finally: - NATIVE_RESPONSES.reset() - assert result is expected - assert [request.resolved["model"] for request in captured] == ["gpt-4o"] - - -@pytest.mark.asyncio -async def test_public_aresponses_routes_through_dispatch(monkeypatch: pytest.MonkeyPatch) -> None: - captured: Final[list[NativeCall]] = [] - expected: Final = _response() - - async def native( - request: NativeCall, - ) -> ResponsesAPIResponse: - captured.append(request) - return expected - - NATIVE_ARESPONSES.override(native) - monkeypatch.setattr(catalog, "RULES", RUST_RULES) - public_aresponses: Final = cast(Callable[..., Awaitable[ResponsesAPIResponse]], litellm.aresponses) - try: - result: Final = await public_aresponses(input=INPUT, model="gpt-4o") - finally: - NATIVE_ARESPONSES.reset() - assert result is expected - assert [request.resolved["model"] for request in captured] == ["gpt-4o"] - - def test_responses_with_retries_uses_the_dispatch_entrypoint(monkeypatch: pytest.MonkeyPatch) -> None: calls: Final[list[Mapping[str, object]]] = [] expected: Final = _response() diff --git a/tests/test_litellm_rust/support/routing.py b/tests/test_litellm_rust/support/routing.py index 4250589848d..e3bcd2ac463 100644 --- a/tests/test_litellm_rust/support/routing.py +++ b/tests/test_litellm_rust/support/routing.py @@ -11,7 +11,7 @@ from tests.test_litellm_rust.support.isolation import rebound from tests.test_litellm_rust.support.recording_server import RecordingServer from tests.test_litellm_rust.support.served import assert_served_natively -__all__ = ["CALL_ID", "assert_served_natively", "call_public", "native_route"] +__all__ = ["CALL_ID", "assert_served_natively", "call_public", "native_route", "routed"] CALL_ID: Final = "callback-contract" @@ -22,6 +22,12 @@ def native_route(*routes: Route) -> Generator[None]: yield +@contextmanager +def routed(*rules: RouteRule) -> Generator[None]: + with rebound(catalog, "RULES", rules): + yield + + async def call_public( create: Callable[..., object], acreate: Callable[..., Awaitable[object]], diff --git a/tests/test_litellm_rust/test_dispatch.py b/tests/test_litellm_rust/test_dispatch.py index 9d53db26ba8..94526e83fbb 100644 --- a/tests/test_litellm_rust/test_dispatch.py +++ b/tests/test_litellm_rust/test_dispatch.py @@ -4,9 +4,8 @@ from typing import Final, TypeAlias import pytest -from litellm.rust_bridge import configuration from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, RouteContext, RouteRule, Rules +from litellm.rust_bridge.catalog import Feature, Route, RouteContext, RouteRule, Rules, Target from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.dispatch import PublicDispatch from litellm.rust_bridge.runtime import NO_PYTHON, NoPythonImplementationError @@ -70,29 +69,6 @@ def test_unconditional_python_rule_prevents_later_rust_rule_projection() -> None assert result is expected -def test_disabled_optional_rust_rule_forwards_before_projection() -> None: - rules: Final[Rules] = (RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),) - - def reject_request(args: tuple[object, ...], kwargs: Mapping[str, object]) -> Request: - pytest.fail("Disabled optional Rust must not project the request") - - dispatch: Final = PublicDispatch(route=Route.OCR, request=reject_request, context=lambda _: RouteContext(Route.OCR)) - expected: Final = object() - configuration.rust(False) - try: - result: Final = dispatch.run( - ("model",), - {}, - python=lambda *args, **kwargs: expected, - binding=binding(), - native=lambda hook, request, args, kwargs: pytest.fail("Disabled optional Rust must not call native"), - rules=rules, - ) - finally: - configuration.rust(None) - assert result is expected - - def test_native_stream_result_is_not_consumed_or_wrapped() -> None: request: Final = Request(model="streaming-model") stream: Final[Iterator[int]] = iter((1, 2)) @@ -207,36 +183,42 @@ def test_rules_for_other_routes_and_constrained_python_rules_skip_projection() - assert result is expected -@pytest.mark.asyncio -async def test_async_bypass_forwards_to_python_without_native() -> None: - request: Final = Request(model="bypassed-model") - rules: Final[Rules] = (RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED),) - dispatch: Final = PublicDispatch( - route=Route.RESPONSES, - request=lambda args, kwargs: request, - context=lambda value: RouteContext(Route.RESPONSES, model=value.model), - bypass=lambda value: value.model == "bypassed-model", - ) +@pytest.mark.parametrize( + "constrained", + ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, providers=frozenset({"other"})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, models=frozenset({"other"})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, targets=frozenset({Target.CHAT_COMPLETIONS})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), + ), + ids=("providers", "models", "targets", "features"), +) +def test_constrained_python_rule_still_projects_for_a_later_rust_rule(constrained: RouteRule) -> None: expected: Final = object() - - async def python(*args: object, **kwargs: object) -> object: # kwargs-ok: public pass-through shape - return expected - - result: Final = await dispatch.arun( - ("bypassed-model",), - {}, - python=python, - binding=binding(), - native=lambda hook, value, args, kwargs: pytest.fail("Bypassed requests must not call native"), - rules=rules, + native_binding: Final[NativeBinding[Callable[[Request], object]]] = NativeBinding("x", validate=lambda _: None) + native_binding.override(lambda request: expected) + dispatch: Final = PublicDispatch( + route=Route.MESSAGES, + request=lambda args, kwargs: Request(model="model"), + context=lambda value: RouteContext(Route.MESSAGES, model=value.model, target=Target.MESSAGES), ) + + result: Final = dispatch.run( + ("model",), + {}, + python=lambda *args, **kwargs: pytest.fail("a constrained Python rule must not hide the Rust rule after it"), + binding=native_binding, + native=lambda hook, value, args, kwargs: hook(value), + rules=(constrained, RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED)), + ) + assert result is expected NativeRoute: TypeAlias = Callable[[Request, tuple[object, ...], Mapping[str, object]], object] -def native_route(result: object) -> NativeBinding[NativeRoute]: +def no_python_binding(result: object) -> NativeBinding[NativeRoute]: bound: Final[NativeBinding[NativeRoute]] = NativeBinding("no_python", validate=lambda _: None) bound.override(lambda request, args, kwargs: (result, request, args, dict(kwargs))) return bound @@ -263,52 +245,43 @@ async def dispatch_without_python( return await dispatch.arun(("model",), {"page": 1}, python=NO_PYTHON, binding=bound, native=native, rules=rules) -def ocr_dispatch(request: Request | None, *, bypass: bool = False) -> PublicDispatch[Request]: +def ocr_dispatch(request: Request | None) -> PublicDispatch[Request]: return PublicDispatch( route=Route.OCR, request=lambda args, kwargs: request, context=lambda value: RouteContext(Route.OCR, model=value.model), - bypass=lambda _: bypass, ) @pytest.mark.parametrize("asynchronous", (False, True)) -@pytest.mark.parametrize("switch", (None, False)) -async def test_dispatch_without_python_hands_every_call_to_native(asynchronous: bool, switch: bool | None) -> None: +async def test_dispatch_without_python_hands_every_call_to_native(asynchronous: bool) -> None: request: Final = Request(model="model") result: Final = object() - configuration.rust(switch) - try: - dispatched: Final = await dispatch_without_python( - ocr_dispatch(request), - native_route(result), - (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),), - asynchronous=asynchronous, - ) - finally: - configuration.rust(None) + dispatched: Final = await dispatch_without_python( + ocr_dispatch(request), + no_python_binding(result), + (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),), + asynchronous=asynchronous, + ) assert dispatched == (result, request, ("model",), {"page": 1}) @pytest.mark.parametrize("asynchronous", (False, True)) @pytest.mark.parametrize( - ("request_value", "bypass", "rules", "reason"), + ("request_value", "rules", "reason"), ( - (Request(model="model"), False, (), "must resolve to RUST_REQUIRED"), - (Request(model="model"), False, (RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),), "must resolve to RUST_REQUIRED"), - (None, False, (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),), "must project to a native request"), - (Request(model="model"), True, (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),), "bypass predicate matches"), + (Request(model="model"), (), "must resolve to RUST_REQUIRED"), + (Request(model="model"), (RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),), "must resolve to RUST_REQUIRED"), + (None, (RouteRule(Route.OCR, Rollout.RUST_REQUIRED),), "must project to a native request"), ), - ids=("no-rule", "opt-out-rule", "unprojectable-call", "bypassed-call"), + ids=("no-rule", "opt-out-rule", "unprojectable-call"), ) async def test_dispatch_without_python_never_falls_back( - asynchronous: bool, request_value: Request | None, bypass: bool, rules: Rules, reason: str + asynchronous: bool, request_value: Request | None, rules: Rules, reason: str ) -> None: bound: Final[NativeBinding[NativeRoute]] = NativeBinding("no_python", validate=lambda _: None) bound.override(lambda request, args, kwargs: pytest.fail("a misdeclared route must not reach native")) with pytest.raises(NoPythonImplementationError, match=f"ocr has no Python implementation, so .*{reason}"): - await dispatch_without_python( - ocr_dispatch(request_value, bypass=bypass), bound, rules, asynchronous=asynchronous - ) + await dispatch_without_python(ocr_dispatch(request_value), bound, rules, asynchronous=asynchronous) diff --git a/tests/test_litellm_rust/test_public_entries.py b/tests/test_litellm_rust/test_public_entries.py new file mode 100644 index 00000000000..2f5c33e3ced --- /dev/null +++ b/tests/test_litellm_rust/test_public_entries.py @@ -0,0 +1,287 @@ +import inspect +from collections.abc import Awaitable, Callable, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass, field +from enum import Enum, auto +from typing import Final, TypeAlias, cast # noqa: TID251 # public entries return union results + +import pytest + +import litellm +from litellm.messages import dispatch as messages +from litellm.rust_bridge.bindings import NativeBinding +from litellm.rust_bridge.catalog import Route, RouteRule +from litellm.rust_bridge.chat_completions.entrypoints import NATIVE_ACOMPLETION, NATIVE_COMPLETION +from litellm.rust_bridge.configuration import Rollout +from litellm.rust_bridge.embeddings.entrypoints import NATIVE_AEMBEDDING, NATIVE_EMBEDDING +from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES +from litellm.rust_bridge.ocr.entrypoints import NATIVE_AOCR, NATIVE_OCR +from litellm.rust_bridge.public_call import NativeCall +from litellm.rust_bridge.responses.entrypoints import NATIVE_ARESPONSES, NATIVE_RESPONSES +from litellm.rust_bridge.runtime import NoPythonImplementationError +from litellm.types.utils import ModelResponse +from tests.test_litellm_rust.support.routing import routed + +MESSAGES: Final = [{"role": "user", "content": "hi"}] + +Served: TypeAlias = Callable[[object], bool] + + +@dataclass(frozen=True, slots=True) +class Entry: + """``python`` is ``None`` for a route with no Python implementation, which only RUST_REQUIRED rules serve.""" + + route: Route + sync: Callable[..., object] + asynchronous: Callable[..., Awaitable[object]] + flag: str + sync_binding: NativeBinding[object] + async_binding: NativeBinding[object] + arguments: Mapping[str, object] + positional: int + python: Served | None + + def args(self) -> tuple[object, ...]: + return tuple(self.arguments.values())[: self.positional] + + def kwargs(self, **extra: object) -> Mapping[str, object]: # kwargs-ok: forwards the async flag + return {**dict(tuple(self.arguments.items())[self.positional :]), **extra} + + +def _binding(value: object) -> NativeBinding[object]: + return cast("NativeBinding[object]", value) # cast-ok: the table only overrides and resets bindings + + +def _chat(result: object) -> bool: + return isinstance(result, ModelResponse) and result.choices[0].message.content == "python" # pyright: ignore[reportAttributeAccessIssue] # non-streaming choice + + +def _embedding(result: object) -> bool: + return getattr(result, "data")[0]["embedding"] == [0.5] # noqa: B009 # union result has no common attribute + + +def _responses(result: object) -> bool: + return getattr(result, "output_text") == "python" # noqa: B009 # union result has no common attribute + + +def _messages(result: object) -> bool: + return isinstance(result, Mapping) and result["content"][0]["text"] == "python" + + +ENTRIES: Final = ( + Entry( + route=Route.CHAT_COMPLETIONS, + sync=litellm.completion, + asynchronous=litellm.acompletion, + flag="acompletion", + sync_binding=_binding(NATIVE_COMPLETION), + async_binding=_binding(NATIVE_ACOMPLETION), + arguments={"model": "openai/gpt-4o", "messages": MESSAGES, "mock_response": "python"}, + positional=2, + python=_chat, + ), + Entry( + route=Route.EMBEDDINGS, + sync=litellm.embedding, + asynchronous=litellm.aembedding, + flag="aembedding", + sync_binding=_binding(NATIVE_EMBEDDING), + async_binding=_binding(NATIVE_AEMBEDDING), + arguments={"model": "openai/text-embedding-3-small", "input": ["hi"], "mock_response": [0.5]}, + positional=2, + python=_embedding, + ), + Entry( + route=Route.RESPONSES, + sync=litellm.responses, + asynchronous=litellm.aresponses, + flag="aresponses", + sync_binding=_binding(NATIVE_RESPONSES), + async_binding=_binding(NATIVE_ARESPONSES), + arguments={"input": "hi", "model": "openai/gpt-4o", "mock_response": "python"}, + positional=2, + python=_responses, + ), + Entry( + route=Route.MESSAGES, + sync=messages.anthropic_messages_handler, + asynchronous=messages.anthropic_messages, + flag="is_async", + sync_binding=_binding(NATIVE_MESSAGES), + async_binding=_binding(NATIVE_AMESSAGES), + arguments={ + "max_tokens": 16, + "messages": MESSAGES, + "model": "anthropic/claude-sonnet-4-5", + "mock_response": "python", + }, + positional=0, + python=_messages, + ), + Entry( + route=Route.OCR, + sync=litellm.ocr, + asynchronous=litellm.aocr, + flag="aocr", + sync_binding=_binding(NATIVE_OCR), + async_binding=_binding(NATIVE_AOCR), + arguments={"model": "mistral/mistral-ocr-latest", "document": {"type": "file", "file": b"pdf"}}, + positional=2, + python=None, + ), +) + + +class Form(Enum): + SYNC = auto() + SYNC_FLAG_FALSE = auto() + ASYNC = auto() + SYNC_FLAG_TRUE = auto() + ASYNC_FLAG_TRUE = auto() + + +async def call(entry: Entry, form: Form) -> object: + match form: + case Form.SYNC: + return entry.sync(*entry.args(), **entry.kwargs()) + case Form.SYNC_FLAG_FALSE: + return entry.sync(*entry.args(), **entry.kwargs(**{entry.flag: False})) + case Form.ASYNC: + return await entry.asynchronous(*entry.args(), **entry.kwargs()) + case Form.SYNC_FLAG_TRUE: + return await cast( + "Awaitable[object]", entry.sync(*entry.args(), **entry.kwargs(**{entry.flag: True})) + ) # cast-ok: a sync entry with its flag set returns a coroutine + case Form.ASYNC_FLAG_TRUE: + return await entry.asynchronous(*entry.args(), **entry.kwargs(**{entry.flag: True})) + + +@dataclass(slots=True) +class Native: + result: ModelResponse = field(default_factory=ModelResponse) + calls: list[tuple[str, NativeCall]] = field(default_factory=list) # mutable-ok: records native calls + + def install(self, entry: Entry) -> None: + def sync(native_call: NativeCall) -> ModelResponse: + self.calls.append(("sync", native_call)) + return self.result + + async def asynchronous(native_call: NativeCall) -> ModelResponse: + self.calls.append(("async", native_call)) + return self.result + + entry.sync_binding.override(sync) + entry.async_binding.override(asynchronous) + + +@pytest.fixture +def native(entry: Entry) -> Iterator[Native]: + recorder: Final = Native() + recorder.install(entry) + yield recorder + entry.sync_binding.reset() + entry.async_binding.reset() + + +class Outcome(Enum): + PYTHON = auto() + NATIVE = auto() + UNAVAILABLE = auto() + NO_PYTHON = auto() + + +@dataclass(frozen=True, slots=True) +class Scenario: + rollout: Rollout | None + binding: bool + outcome: Outcome + + +SCENARIOS: Final = { + "no-rule": Scenario(None, binding=True, outcome=Outcome.PYTHON), + "python-only": Scenario(Rollout.PYTHON_ONLY, binding=True, outcome=Outcome.PYTHON), + "required": Scenario(Rollout.RUST_REQUIRED, binding=True, outcome=Outcome.NATIVE), + "required-without-extension": Scenario(Rollout.RUST_REQUIRED, binding=False, outcome=Outcome.UNAVAILABLE), +} + + +def expected(entry: Entry, scenario: Scenario) -> Outcome: + if entry.python is None and scenario.rollout is not Rollout.RUST_REQUIRED: + return Outcome.NO_PYTHON + return scenario.outcome + + +@contextmanager +def scenario_rules(entry: Entry, scenario: Scenario) -> Iterator[None]: + if not scenario.binding: + entry.sync_binding.override(None) + entry.async_binding.override(None) + with routed(*(() if scenario.rollout is None else (RouteRule(entry.route, scenario.rollout),))): + yield + + +def entry_id(entry: Entry) -> str: + return entry.route.value + + +@pytest.mark.parametrize("form", tuple(Form), ids=lambda form: form.name.lower()) +@pytest.mark.parametrize("scenario", tuple(SCENARIOS.values()), ids=tuple(SCENARIOS)) +@pytest.mark.parametrize("entry", ENTRIES, ids=entry_id) +async def test_every_public_form_takes_the_path_the_rules_pick( + entry: Entry, scenario: Scenario, form: Form, native: Native +) -> None: + outcome: Final = expected(entry, scenario) + with scenario_rules(entry, scenario): + if outcome is Outcome.UNAVAILABLE: + with pytest.raises(RuntimeError, match=f"Rust {entry.route.value} bridge is unavailable"): + await call(entry, form) + return + if outcome is Outcome.NO_PYTHON: + with pytest.raises(NoPythonImplementationError, match=f"{entry.route.value} has no Python implementation"): + await call(entry, form) + assert native.calls == [] + return + result: Final = await call(entry, form) + + if outcome is Outcome.PYTHON: + assert entry.python is not None + assert entry.python(result) + assert native.calls == [] + return + assert result is native.result + [(binding, native_call)] = native.calls + assert binding == ("sync" if form in (Form.SYNC, Form.SYNC_FLAG_FALSE) else "async") + assert {name: native_call.resolved[name] for name in entry.arguments} == entry.arguments + assert entry.flag not in native_call.resolved + + +@pytest.mark.parametrize("entry", ENTRIES, ids=entry_id) +async def test_async_forms_hand_native_the_same_call(entry: Entry, native: Native) -> None: + with scenario_rules(entry, SCENARIOS["required"]): + for form in (Form.ASYNC, Form.SYNC_FLAG_TRUE, Form.ASYNC_FLAG_TRUE): + _ = await call(entry, form) + + assert [binding for binding, _ in native.calls] == ["async", "async", "async"] + resolved: Final = [dict(native_call.resolved) for _, native_call in native.calls] + assert resolved[1:] == [resolved[0], resolved[0]] + + +@pytest.mark.parametrize("entry", ENTRIES, ids=entry_id) +async def test_sync_entry_with_flag_true_does_nothing_until_awaited(entry: Entry, native: Native) -> None: + with scenario_rules(entry, SCENARIOS["required"]): + pending: Final = entry.sync(*entry.args(), **entry.kwargs(**{entry.flag: True})) + + assert inspect.iscoroutine(pending) + assert native.calls == [] + assert await pending is native.result + assert len(native.calls) == 1 + + +@pytest.mark.parametrize("entry", ENTRIES, ids=entry_id) +async def test_sync_entry_with_flag_true_raises_setup_errors_at_await(entry: Entry, native: Native) -> None: + with scenario_rules(entry, SCENARIOS["required-without-extension"]): + pending: Final = entry.sync(*entry.args(), **entry.kwargs(**{entry.flag: True})) + + assert inspect.iscoroutine(pending) + with pytest.raises(RuntimeError, match="is unavailable"): + await pending diff --git a/tests/unit/rust_bridge/AGENTS.md b/tests/unit/rust_bridge/AGENTS.md index 64c9fbbadd3..abf5ac6bed7 100644 --- a/tests/unit/rust_bridge/AGENTS.md +++ b/tests/unit/rust_bridge/AGENTS.md @@ -4,6 +4,6 @@ Test what each side of the bridge does, not the rollout policy that picks a side Call each path directly with an explicit decision instead. The Python path is the implementation the dispatcher falls back to, e.g. `litellm.responses.main.responses`. The Rust path is the native binding, e.g. `NATIVE_OCR.load()` from `litellm/rust_bridge/ocr/entrypoints.py`, called with the `NativeCall` envelope that dispatch would hand it for every public inference route. When the native side reads a policy-derived setting such as `settings.secret_manager().native`, pin that field in the test instead of deriving it from the catalog. `ocr/test_secrets.py` shows the pattern -Rollout policy itself, meaning which rule matches and what `LITELLM_RUST` changes, belongs in `test_catalog.py`, `test_configuration.py` and `test_dispatch.py`, tested against rules the test builds rather than the shipped `catalog.RULES` +Rollout policy itself, meaning which rule matches and what `LITELLM_RUST` changes, belongs in `test_catalog.py` and `test_configuration.py`, tested against rules the test builds rather than the shipped `catalog.RULES`. Which path a public call takes under those rules is a dispatch test and lives in `tests/test_litellm_rust/` for now Before adding a test here, ask whether it checks something Rust cannot. A `host/routes/.py` module is the Python half of a native route: it projects Python-only state (the cost map, `litellm.*` settings, request kwargs) into the plain values the Rust side consumes, and maps native failures back onto public exceptions. Those projections are what belongs here, because a wrong key or an ignored provider prefix ships the wrong value to Rust and no Rust test sees it. `host/routes/test_messages.py` shows the shape. Behavior that lives in Rust (a request transform given its inputs, header assembly, stream relay) is tested in the crate, and the route end to end is tested against a recording server in `tests/test_litellm_rust/`. A test that only re-checks a Python helper the route host happens to call is a duplicate of that helper's own test and should not be added diff --git a/tests/unit/rust_bridge/test_catalog.py b/tests/unit/rust_bridge/test_catalog.py index b99342388d6..704a4f73fe8 100644 --- a/tests/unit/rust_bridge/test_catalog.py +++ b/tests/unit/rust_bridge/test_catalog.py @@ -7,10 +7,12 @@ import pytest from litellm.rust_bridge import catalog, configuration from litellm.rust_bridge.catalog import ( + Feature, Route, RouteContext, RouteRule, Rules, + Target, ) from litellm.rust_bridge.configuration import Decision, Rollout @@ -23,36 +25,6 @@ def isolated_configuration(monkeypatch: pytest.MonkeyPatch) -> Generator[None]: configuration.reset_rust_configuration() -@pytest.mark.parametrize("route", tuple(Route)) -@pytest.mark.parametrize( - "provider", (None, "bedrock", "mistral", "anthropic", "deepseek", "vertex_ai", "openai", "azure_ai", "unknown") -) -@pytest.mark.parametrize("process", (None, False, True)) -@pytest.mark.parametrize("environment", (None, "0", "1")) -def test_shipped_decisions( - monkeypatch: pytest.MonkeyPatch, - route: Route, - provider: str | None, - process: bool | None, - environment: str | None, -) -> None: - configuration.rust(process) - if environment is not None: - monkeypatch.setenv("LITELLM_RUST", environment) - context: Final = RouteContext(route, provider=provider, model="test-model") - - if route is Route.OCR or (route is Route.TRANSCRIPTION and provider == "bedrock"): - assert catalog.rollout(context) is Rollout.RUST_REQUIRED - assert catalog.decision(context) is Decision.RUST_REQUIRED - elif route is Route.MESSAGES and provider in ("anthropic", "deepseek", "vertex_ai"): - assert catalog.rollout(context) is Rollout.RUST_OPT_IN - opted_in: Final = environment == "1" or (environment is None and process is True) - assert catalog.decision(context) is (Decision.RUST_WITH_FALLBACK if opted_in else Decision.PYTHON) - else: - assert catalog.rollout(context) is Rollout.PYTHON_ONLY - assert catalog.decision(context) is Decision.PYTHON - - @pytest.mark.parametrize("route", tuple(Route)) def test_missing_rule_stays_on_python_even_when_rust_is_enabled(monkeypatch: pytest.MonkeyPatch, route: Route) -> None: configuration.rust(True) @@ -72,6 +44,11 @@ def test_missing_rule_stays_on_python_even_when_rust_is_enabled(monkeypatch: pyt (RouteContext(Route.RESPONSES, provider="openai", model="other"), Decision.PYTHON), (RouteContext(Route.RESPONSES, provider="anthropic", model="m"), Decision.PYTHON), (RouteContext(Route.MESSAGES, provider="openai", model="m"), Decision.PYTHON), + (RouteContext(Route.MESSAGES, target=Target.MESSAGES), Decision.RUST_REQUIRED), + (RouteContext(Route.MESSAGES, target=Target.CHAT_COMPLETIONS), Decision.PYTHON), + (RouteContext(Route.MESSAGES), Decision.PYTHON), + (RouteContext(Route.OCR, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), Decision.PYTHON), + (RouteContext(Route.OCR), Decision.RUST_REQUIRED), ), ) def test_first_matching_rule_respects_every_constraint(context: RouteContext, expected: Decision) -> None: @@ -83,24 +60,14 @@ def test_first_matching_rule_respects_every_constraint(context: RouteContext, ex models=frozenset({"m"}), ), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED, targets=frozenset({Target.MESSAGES})), + RouteRule(Route.OCR, Rollout.PYTHON_ONLY, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED), ) assert catalog.decision(context, rules) is expected -@pytest.mark.parametrize("process", (None, False, True)) -@pytest.mark.parametrize("environment", (None, "0", "1")) -def test_ocr_has_no_python_path_to_opt_out_to( - monkeypatch: pytest.MonkeyPatch, process: bool | None, environment: str | None -) -> None: - configuration.rust(process) - if environment is not None: - monkeypatch.setenv("LITELLM_RUST", environment) - - assert catalog.decision(RouteContext(Route.OCR, model="m")) is Decision.RUST_REQUIRED - assert catalog.decision(RouteContext(Route.OCR, provider="aws_textract", model="m")) is Decision.RUST_REQUIRED - - @pytest.mark.parametrize( ("context", "expected"), ( @@ -154,6 +121,10 @@ def test_empty_constraints_match_nothing() -> None: rules: Final[Rules] = ( RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset()), RouteRule(Route.OCR, Rollout.RUST_REQUIRED, models=frozenset()), + RouteRule(Route.OCR, Rollout.RUST_REQUIRED, features=frozenset()), ) assert catalog.decision(RouteContext(Route.OCR), rules) is Decision.PYTHON + assert catalog.decision( + RouteContext(Route.OCR, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), rules + ) is Decision.PYTHON diff --git a/tests/unit/rust_bridge/test_public_call.py b/tests/unit/rust_bridge/test_public_call.py index 63029e288e2..5f50db6ea5a 100644 --- a/tests/unit/rust_bridge/test_public_call.py +++ b/tests/unit/rust_bridge/test_public_call.py @@ -3,7 +3,7 @@ from typing import Final import pytest -from litellm.rust_bridge.public_call import native_call, signature +from litellm.rust_bridge.public_call import AsyncFlag, native_call, signature def _messages( @@ -18,6 +18,7 @@ def _messages( _SIGNATURE: Final = signature(_messages) +_FLAG: Final = AsyncFlag("aflag", sync=_SIGNATURE) @pytest.mark.parametrize("supplied", ({}, {"api_key": None}, {"api_key": "explicit"})) @@ -62,3 +63,16 @@ def test_resolved_lays_the_keywords_over_the_base_and_equals_the_full_binding() } assert call.resolved["metadata"] is metadata assert supplied == {"model": "keyword-model", "metadata": metadata} + + +def test_flag_keywords_name_positionals_and_drop_only_the_flag() -> None: + messages: Final[Sequence[object]] = [{"role": "user", "content": "hello"}] + + keywords: Final = _FLAG.keywords((128, messages), {"model": "m", "stream": True, "aflag": True}) + + assert keywords == {"max_tokens": 128, "messages": messages, "model": "m", "stream": True} + + +def test_flag_keywords_reject_a_parameter_given_twice() -> None: + with pytest.raises(TypeError): + _FLAG.keywords((128,), {"max_tokens": 64}) From 730db4d22854465d92f0ac9f884fafcee31f6d13 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Tue, 20 Oct 2026 15:06:25 -0700 Subject: [PATCH 006/235] refactor(rust-bridge): detect Python-only features in the catalog instead of Rust declines (#46714) * refactor(rust-bridge): detect Python-only features in the catalog instead of Rust declines * refactor(rust-bridge): split and document route-specific detectors --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../crates/python-bridge/src/cache/mod.rs | 2 +- .../python-bridge/src/cache/native/v2.rs | 2 +- .../python-bridge/src/cache/selection.rs | 15 - .../python-bridge/src/routes/inference.rs | 1 - .../python-bridge/src/routes/responses/mod.rs | 40 -- litellm/chat_completions/dispatch.py | 26 +- .../messages/interceptors/advisor.py | 4 +- .../messages/interceptors/base.py | 4 +- litellm/messages/dispatch.py | 33 +- litellm/responses/dispatch.py | 55 +- litellm/rust_bridge/AGENTS.md | 54 +- litellm/rust_bridge/catalog.py | 137 ++++- .../rust_bridge/chat_completions/detectors.py | 137 +++++ litellm/rust_bridge/detectors.py | 183 ++++++ litellm/rust_bridge/dispatch.py | 20 +- litellm/rust_bridge/host/routes/responses.py | 14 +- litellm/rust_bridge/messages/detectors.py | 85 +++ litellm/rust_bridge/public_call.py | 40 -- litellm/rust_bridge/responses/detectors.py | 210 +++++++ tests/test_litellm_rust/AGENTS.md | 85 ++- .../chat_completions/test_dispatch.py | 91 ++- .../messages/test_dispatch.py | 176 ++++++ tests/test_litellm_rust/responses/data.py | 4 +- .../responses/test_dispatch.py | 170 +++++- .../responses/test_failures.py | 25 - tests/test_litellm_rust/test_dispatch.py | 46 +- tests/unit/rust_bridge/test_detectors.py | 522 ++++++++++++++++++ 27 files changed, 1980 insertions(+), 201 deletions(-) create mode 100644 litellm/rust_bridge/chat_completions/detectors.py create mode 100644 litellm/rust_bridge/detectors.py create mode 100644 litellm/rust_bridge/messages/detectors.py create mode 100644 litellm/rust_bridge/responses/detectors.py create mode 100644 tests/unit/rust_bridge/test_detectors.py diff --git a/litellm-rust/crates/python-bridge/src/cache/mod.rs b/litellm-rust/crates/python-bridge/src/cache/mod.rs index 5a8f9bcf94f..e190b12e665 100644 --- a/litellm-rust/crates/python-bridge/src/cache/mod.rs +++ b/litellm-rust/crates/python-bridge/src/cache/mod.rs @@ -4,7 +4,7 @@ mod selection; pub(crate) use native::NativeCacheHandle; pub(crate) use python::{CacheCall, PythonCache}; -pub(crate) use selection::{Cached, Selection, admit_native, configure, configured_native}; +pub(crate) use selection::{Cached, Selection, configure, configured_native}; use litellm_cache::Error; use pyo3::{ diff --git a/litellm-rust/crates/python-bridge/src/cache/native/v2.rs b/litellm-rust/crates/python-bridge/src/cache/native/v2.rs index 82771644351..eb09b8131c3 100644 --- a/litellm-rust/crates/python-bridge/src/cache/native/v2.rs +++ b/litellm-rust/crates/python-bridge/src/cache/native/v2.rs @@ -279,7 +279,7 @@ pub(in crate::cache) fn configured( )> { let handle = native_handle(configured)?.ok_or_else(|| { pyo3::exceptions::PyRuntimeError::new_err( - "the configured cache changed to a Python cache after native admission", + "native inference needs a v2 cache, the catalog keeps Python caches on Python", ) })?; let cache = handle.extract::>()?; diff --git a/litellm-rust/crates/python-bridge/src/cache/selection.rs b/litellm-rust/crates/python-bridge/src/cache/selection.rs index b2dc67fe965..cd19533fc73 100644 --- a/litellm-rust/crates/python-bridge/src/cache/selection.rs +++ b/litellm-rust/crates/python-bridge/src/cache/selection.rs @@ -76,21 +76,6 @@ fn selected_cache<'py>( Ok(Some(configured)) } -pub(crate) fn admit_native( - py: Python<'_>, - kwargs: &Bound<'_, PyDict>, - call_type: &str, -) -> PyResult<()> { - if let Some(configured) = selected_cache(py, kwargs, call_type)? - && native::v2::native_handle(&configured)?.is_none() - { - return Err(crate::errors::RustBridgeDeclined::new_err( - "the configured cache requires Python inference", - )); - } - Ok(()) -} - pub(crate) fn configured_native( py: Python<'_>, kwargs: &Bound<'_, PyDict>, diff --git a/litellm-rust/crates/python-bridge/src/routes/inference.rs b/litellm-rust/crates/python-bridge/src/routes/inference.rs index a5be7f51828..3b590a522fe 100644 --- a/litellm-rust/crates/python-bridge/src/routes/inference.rs +++ b/litellm-rust/crates/python-bridge/src/routes/inference.rs @@ -177,7 +177,6 @@ where } else { R::SYNC_CALL_TYPE }; - crate::cache::admit_native(py, &call.kwargs, call_type)?; let (arguments, hooks) = super::call_hooks(py, R::OPERATION, &call, asynchronous)?; super::run_public_call( py, diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs index 9caeae6c3da..9c045d65c13 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -8,59 +8,19 @@ use litellm_auth::AuthServices; use litellm_cache_response::{CachePolicy, ScopedCache}; use litellm_callbacks_legacy_python::LoggingOperation; use litellm_host::{call::HostedMachine, protocol::Protocol}; -use litellm_host_python::present; use litellm_inference_responses::{ResponsesRoute, route::Responses}; use litellm_secrets::source::SecretSource; use pyo3::prelude::*; -use serde_json::Value; pub(crate) use websocket::ResponsesWebSocketConnection; use super::{ NativeCall, inference::{InferenceHost, InferenceRoute, run_inference}, }; -use crate::errors::RustBridgeDeclined; const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.host.routes.responses"; fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult> { - let resolved = call.resolved()?; - if let Some(reason) = py - .import(ROUTE_HOST_MODULE)? - .getattr("decline_reason")? - .call1((&resolved,))? - .extract::>()? - { - return Err(RustBridgeDeclined::new_err(reason)); - } - let argument = |name: &str| present(&call.kwargs, &call.base, name); - let model = argument("model")? - .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("model is required"))? - .extract::()?; - let provider = argument("custom_llm_provider")? - .map(|value| value.extract::()) - .transpose()?; - if provider - .as_deref() - .is_some_and(|provider| provider != "openai") - || model - .strip_prefix("openai/") - .unwrap_or(&model) - .contains('/') - { - return Err(RustBridgeDeclined::new_err( - "native HTTP responses provider", - )); - } - if argument("stream")? - .map(|value| litellm_host_python::from_py::(&value)) - .transpose()? - .is_some_and(|value| value == Value::Bool(true)) - { - return Err(RustBridgeDeclined::new_err( - "native Python responses streaming", - )); - } let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE); run_inference::(py, call, asynchronous, ResponsesPythonHost::new(host)) } diff --git a/litellm/chat_completions/dispatch.py b/litellm/chat_completions/dispatch.py index cfdcb76a26d..06ba8e9ad76 100644 --- a/litellm/chat_completions/dispatch.py +++ b/litellm/chat_completions/dispatch.py @@ -1,9 +1,12 @@ import inspect from collections.abc import Awaitable, Callable, Coroutine, Mapping +from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable from litellm import main -from litellm.rust_bridge.catalog import Route, RouteContext +from litellm.rust_bridge import detectors +from litellm.rust_bridge.catalog import Feature, Route, RouteContext +from litellm.rust_bridge.chat_completions import detectors as route_detectors from litellm.rust_bridge.chat_completions.entrypoints import ( NATIVE_ACOMPLETION, NATIVE_COMPLETION, @@ -63,11 +66,32 @@ def _public_request( return native_call(legacy, args, kwargs) +_FEATURES: Final[detectors.Detectors] = MappingProxyType( + { + **detectors.COMMON, + Feature.MOCK_RESPONSE: route_detectors.chat_mock_response, + Feature.AGENTIC_LOOP_HOOK: route_detectors.chat_agentic_loop_hook, + Feature.RESPONSE_CACHE: detectors.response_cache, + Feature.STREAMING: detectors.streaming, + Feature.DROP_PARAMS: detectors.drop_params, + Feature.MODIFY_PARAMS: detectors.modify_params, + Feature.EXTRA_BODY: detectors.extra_body, + Feature.RETRIES_FALLBACKS: route_detectors.chat_retries_fallbacks, + Feature.PROMPT_MANAGEMENT: detectors.prompt_management, + Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, + Feature.MODEL_PARAM_GATING: route_detectors.chat_model_param_gating, + Feature.MODEL_LIST: route_detectors.model_list, + Feature.MODEL_ALIAS_MAP: route_detectors.model_alias_map, + } +) + + def _context(request: NativeCall) -> RouteContext: return RouteContext( Route.CHAT_COMPLETIONS, provider=optional_str(request.resolved.get("custom_llm_provider")), model=str(request.resolved["model"]), + features=detectors.detected(_FEATURES, request), ) diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py index 03b27201398..85d76de5402 100644 --- a/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/pass_through/messages/interceptors/advisor.py @@ -15,7 +15,7 @@ How it works: """ import uuid -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final import litellm @@ -48,7 +48,7 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): def can_handle( self, - tools: list[dict] | None, + tools: Sequence[Mapping[str, object]] | None, custom_llm_provider: str | None, ) -> bool: if not tools: diff --git a/litellm/llms/anthropic/pass_through/messages/interceptors/base.py b/litellm/llms/anthropic/pass_through/messages/interceptors/base.py index c32b21564a7..3692c03b6a5 100644 --- a/litellm/llms/anthropic/pass_through/messages/interceptors/base.py +++ b/litellm/llms/anthropic/pass_through/messages/interceptors/base.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping, Sequence from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -21,7 +21,7 @@ class MessagesInterceptor(ABC): @abstractmethod def can_handle( self, - tools: list[dict] | None, + tools: Sequence[Mapping[str, object]] | None, custom_llm_provider: str | None, ) -> bool: """Return True if this interceptor should handle the request.""" diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 4851d7e01b4..cdb9a925403 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -6,9 +6,10 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.llms.anthropic.pass_through.messages import handler as main -from litellm.llms.anthropic.pass_through.safeguards import native_route_classifies +from litellm.rust_bridge import detectors from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target from litellm.rust_bridge.dispatch import PublicDispatch +from litellm.rust_bridge.messages import detectors as route_detectors from litellm.rust_bridge.messages.entrypoints import ( NATIVE_AMESSAGES, NATIVE_MESSAGES, @@ -99,16 +100,30 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target: return Target.CHAT_COMPLETIONS -_FEATURES: Final[Mapping[Feature, Callable[[NativeCall], bool]]] = MappingProxyType( +_FEATURES: Final[detectors.Detectors] = MappingProxyType( { - Feature.SAFEGUARDS_CLASSIFIER: lambda request: native_route_classifies( - str(request.resolved["model"]), request.resolved.get("safeguards") - ), + **detectors.COMMON, + Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier, + Feature.RESPONSE_CACHE: detectors.response_cache, + Feature.MODIFY_PARAMS: detectors.modify_params, + Feature.WORKLOAD_IDENTITY: route_detectors.anthropic_workload_identity, + Feature.CACHE_CONTROL_INJECTION: detectors.cache_control_injection, + Feature.STREAM_COST_INJECTION: route_detectors.stream_cost_injection, + Feature.THINKING_SIGNATURE_RETRY: route_detectors.thinking_signature_retry, + } +) + +_ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( + { + **_FEATURES, + Feature.PRE_REQUEST_HOOK: route_detectors.pre_request_hook, + Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, + Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, } ) -def _context(request: NativeCall) -> RouteContext: +def _context(request: NativeCall, features: detectors.Detectors) -> RouteContext: requested_model: Final = str(request.resolved["model"]) model, provider = _resolved(request) return RouteContext( @@ -116,20 +131,20 @@ def _context(request: NativeCall) -> RouteContext: provider=provider, model=requested_model, target=_target(request, model, provider), - features=frozenset(feature for feature, detected in _FEATURES.items() if detected(request)), + features=detectors.detected(features, request), ) _DISPATCH: Final = PublicDispatch( route=Route.MESSAGES, request=lambda args, kwargs: _public_request(_MESSAGES, args, kwargs), - context=_context, + context=lambda request: _context(request, _FEATURES), ) _ADISPATCH: Final = PublicDispatch( route=Route.MESSAGES, request=lambda args, kwargs: _public_request(_AMESSAGES, args, kwargs), - context=_context, + context=lambda request: _context(request, _ASYNC_FEATURES), ) diff --git a/litellm/responses/dispatch.py b/litellm/responses/dispatch.py index 281aa177bae..0dda3f93660 100644 --- a/litellm/responses/dispatch.py +++ b/litellm/responses/dispatch.py @@ -1,10 +1,14 @@ import inspect from collections.abc import Awaitable, Callable, Coroutine, Mapping +from types import MappingProxyType from typing import Final, TypeAlias, cast # noqa: TID251 # native binding selects a sync result or an async awaitable +from litellm.exceptions import BadRequestError +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.responses import main from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator -from litellm.rust_bridge.catalog import Route, RouteContext +from litellm.rust_bridge import detectors +from litellm.rust_bridge.catalog import Feature, Route, RouteContext from litellm.rust_bridge.dispatch import PublicDispatch from litellm.rust_bridge.public_call import ( AsyncFlag, @@ -15,6 +19,7 @@ from litellm.rust_bridge.public_call import ( optional_str, signature, ) +from litellm.rust_bridge.responses import detectors as route_detectors from litellm.rust_bridge.responses.entrypoints import ( NATIVE_ARESPONSES, NATIVE_RESPONSES, @@ -61,24 +66,64 @@ def _public_request( return native_call(legacy, args, kwargs) -def _context(request: NativeCall) -> RouteContext: +def _provider(request: NativeCall) -> str | None: + provider: Final = optional_str(request.resolved.get("custom_llm_provider")) + try: + return get_llm_provider(str(request.resolved["model"]), provider)[1] + except BadRequestError: + return provider + + +_FEATURES: Final[detectors.Detectors] = MappingProxyType( + { + **detectors.COMMON, + Feature.DROP_PARAMS: detectors.drop_params, + Feature.MODIFY_PARAMS: detectors.modify_params, + Feature.STREAMING: detectors.streaming, + Feature.PROMPT_MANAGEMENT: detectors.prompt_management, + Feature.TEXT_FORMAT: route_detectors.text_format, + Feature.CHAT_COMPLETIONS_BRIDGE: route_detectors.chat_completions_bridge, + Feature.RETRIES_FALLBACKS: detectors.retries_fallbacks, + Feature.DEPLOYMENT_RESPONSE_ID: route_detectors.deployment_response_id, + Feature.ENCODED_ITEM_IDS: route_detectors.encoded_item_ids, + Feature.INPUT_REWRITE: route_detectors.input_rewrite, + Feature.EXTRA_BODY: detectors.extra_body, + Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity, + Feature.RESPONSE_CACHE: detectors.response_cache, + Feature.MANAGED_RESOURCE_IDS: route_detectors.managed_resource_ids, + Feature.CACHE_CONTROL_MARKS: route_detectors.cache_control_marks, + Feature.ANTHROPIC_TOOL_SEARCH: route_detectors.anthropic_tool_search, + Feature.TOOL_SCHEMA_REPAIR: route_detectors.tool_schema_repair, + Feature.MODEL_PARAM_GATING: route_detectors.model_param_gating, + Feature.WORKLOAD_IDENTITY: route_detectors.openai_workload_identity, + Feature.CLIENT_HEADERS: route_detectors.client_headers, + } +) + +_SYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( + {**_FEATURES, Feature.DEPLOYMENT_HOOK: route_detectors.deployment_hook} +) + + +def _context(request: NativeCall, features: detectors.Detectors) -> RouteContext: return RouteContext( Route.RESPONSES, - provider=optional_str(request.resolved.get("custom_llm_provider")), + provider=_provider(request), model=str(request.resolved["model"]), + features=detectors.detected(features, request), ) _DISPATCH: Final = PublicDispatch( route=Route.RESPONSES, request=lambda args, kwargs: _public_request(_RESPONSES, args, kwargs), - context=_context, + context=lambda request: _context(request, _SYNC_FEATURES), ) _ADISPATCH: Final = PublicDispatch( route=Route.RESPONSES, request=lambda args, kwargs: _public_request(_ARESPONSES, args, kwargs), - context=_context, + context=lambda request: _context(request, _FEATURES), ) diff --git a/litellm/rust_bridge/AGENTS.md b/litellm/rust_bridge/AGENTS.md index 6c072684f84..c197b21bad0 100644 --- a/litellm/rust_bridge/AGENTS.md +++ b/litellm/rust_bridge/AGENTS.md @@ -1,21 +1,53 @@ -# Python boundary +# Ownership -This package owns native rollout and fallback selection, Python public API compatibility, settings projection and the Python bindings supplied to the Rust bridge. Rust core owns provider execution. `litellm-host-python` owns CPython runtime mechanics. `callbacks-legacy-python` owns legacy callback sharing and dispatch policy +- This package owns + - choosing Rust or Python for each call, and falling back to Python when Rust is unavailable + - Python public API compatibility at the native boundary + - projecting Python settings into the values Rust reads + - the Python bindings handed to the Rust bridge +- It does not own + - provider execution, which is Rust core + - CPython runtime mechanics, which is `litellm-host-python` + - legacy callback sharing and dispatch policy, which is `callbacks-legacy-python` +- `host/` holds the modules Rust imports by name to read Python-owned state. `host/AGENTS.md` states that contract -`host/` holds the Python modules Rust imports by name to read Python-owned state (provider resolution, model capabilities, settings, caches, diagnostics, legacy callbacks, the secret manager, transport and timeout conversions). Its AGENTS.md states that contract +# Routing -`execution.py` owns generic inline execution and stream iteration, plus the step classes Rust constructs. Native entrypoints return an unstarted `Execution`, and `execution.synchronous` or `execution.asynchronous` hands it to the public caller through `run` or `drive` with the one stream pair, so Rust never names a Python driver or stream class. Keep one driver implementing generator-style `send` (`send(None)` starts), `throw` and idempotent `close`. Do not create a second implementation +- `catalog.py` decides, the detectors describe the request, `dispatch.py` applies the decision +- Shared detectors live in `detectors.py`; route-only detectors live in `/detectors.py` +- The first matching rule wins. A context no rule matches stays on Python +- Every Python-only exclusion for a route (a `targets` or `features` rule) sits above any Rust rule for that route +- A route whose Python side is `NO_PYTHON` resolves to `RUST_REQUIRED` for every context it can produce + - `tests/test_litellm_rust/test_public_entries.py` marks such a route `python=None`, so dropping a route's Python changes its entries, its rules and that field together +- A detector reads the same inputs the legacy Python path reads, so the catalog and the fallback agree on which requests need Python + - A route's context, its target as well as its features, reads `NativeCall.resolved`, so an argument counts the same whether it came by position, by keyword or as a signature default + - A route's sync and async dispatches read different detector maps when the legacy handlers differ: Messages runs pre-request hooks, web search interception and interceptors only on async calls, and the sync Responses handler runs the deployment hook itself +- A detector that raises keeps the call on Python. `PublicDispatch` logs it and never fails the call over routing +- `catalog.RULES` is rollout policy with no tests of its own. Tests pass the rules they need, so check these invariants by review whenever `RULES` changes -Generic execution steps and inline suspension handling must not depend on LiteLLM response metadata. Public stream construction, `_hidden_params` and header compatibility remain product responsibilities. Keep generic stream heads opaque and preserve public header behavior in the product wrappers +# Public entries -Await each selected suspension in the caller's task so hooks retain thread, loop and context identity. A final awaitable value is returned as a value, not awaited implicitly. Closing, cancellation and `GeneratorExit` release the execution without replaying provider work or starting further callbacks +- Every route serves `a(...)`, `(..., a=True)` and `a(..., a=True)` the same way, handing native one identical `NativeCall` with no async flag on the async binding +- `tests/test_litellm_rust/test_public_entries.py` holds this for every route in its table, so a new public route gets a row there -Native admission may select legacy before execution starts. A failure after execution starts is terminal and never authorizes fallback. Creating an unstarted native coroutine must not acquire clients or credentials or capture execution context +# Execution -Verify cross-language behavior using a fresh installed extension, including unstarted-call release, setup ordering, exception identity, sync and async streams, cancellation and GC. Keep `_native.pyi` accurate about Future-returning and coroutine-returning APIs +- `execution.py` owns generic inline execution, stream iteration and the step classes Rust constructs +- Native entrypoints return an unstarted `Execution`. `execution.synchronous` or `execution.asynchronous` hands it to the caller through `run` or `drive`, so Rust never names a Python driver or stream class +- Keep one driver, implementing generator-style `send` (`send(None)` starts), `throw` and idempotent `close` +- Generic steps and suspension handling never depend on LiteLLM response metadata. Public stream construction, `_hidden_params` and header compatibility stay in the product wrappers, and generic stream heads stay opaque +- Await each selected suspension in the caller's task, so hooks keep their thread, loop and context +- Return a final awaitable as a value instead of awaiting it implicitly +- Closing, cancellation and `GeneratorExit` release the execution without replaying provider work or starting more callbacks -A route with no Python implementation passes `runtime.NO_PYTHON` from both public entries, and every catalog rule for it must be `RUST_REQUIRED`. Dispatch raises `NoPythonImplementationError` for any other rollout instead of falling back. `tests/test_litellm_rust/test_public_entries.py` marks such a route with `python=None`, so dropping a route's Python changes the entries, its rules and that one table field together +# Fallback -A route's context, its target as well as its features, is built from `NativeCall.resolved`, so an argument counts the same whether it came by position, by keyword or as a signature default +- Native admission may pick Python before execution starts +- A failure after execution starts is terminal and never triggers fallback +- Creating an unstarted native coroutine acquires no clients or credentials and captures no execution context -Every public route serves `a(...)`, `(..., a=True)` and `a(..., a=True)` the same way. Each hands native one identical `NativeCall` with no async flag, on the async binding. `test_public_entries.py` holds this for every route in its table, so a new public route gets a row there +# Verification + +- Check cross-language behavior against a freshly installed extension, covering unstarted-call release, setup ordering, exception identity, sync and async streams, cancellation and GC +- Keep `_native.pyi` accurate about which APIs return a Future and which return a coroutine +- Unit tests of this package live in `tests/unit/rust_bridge/`, end-to-end Rust path tests in `tests/test_litellm_rust/` diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 37548943c80..ebb221e98be 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -33,6 +33,77 @@ class Target(str, Enum): class Feature(str, Enum): SAFEGUARDS_CLASSIFIER = "safeguards_classifier" + """A non-Claude model with dangerous_tool_use safeguards, which the proxy classifier answers.""" + MOCK_RESPONSE = "mock_response" + """mock_response short-circuits before any provider call.""" + PROXY_MCP_TOOLS = "proxy_mcp_tools" + """litellm_proxy MCP tools that Python expands and may auto-execute.""" + CALLER_CLIENT = "caller_client" + """A caller-supplied HTTP client or shared session that Python sends through.""" + TIMEOUT_VARIANTS = "timeout_variants" + """request_timeout, stream_timeout or httpx.Timeout phases beyond a plain timeout.""" + DROP_PARAMS = "drop_params" + """drop_params or additional_drop_params dropping fields per provider.""" + MODIFY_PARAMS = "modify_params" + """litellm.modify_params rewriting the request, like clamping max_tokens.""" + STREAMING = "streaming" + """stream=True.""" + PROMPT_MANAGEMENT = "prompt_management" + """prompt_id resolved through a prompt manager.""" + TEXT_FORMAT = "text_format" + """A text_format model that Python turns into a JSON schema and parses back.""" + CHAT_COMPLETIONS_BRIDGE = "chat_completions_bridge" + """use_chat_completions_api or the openai/chat_completions/ prefix, served through Chat Completions.""" + RETRIES_FALLBACKS = "retries_fallbacks" + """num_retries, max_retries or fallbacks run by the SDK.""" + DEPLOYMENT_RESPONSE_ID = "deployment_response_id" + """A deployment id that Python encodes into the returned response id.""" + ENCODED_ITEM_IDS = "encoded_item_ids" + """Input items carrying litellm-encoded ids that Python restores before sending.""" + INPUT_REWRITE = "input_rewrite" + """Responses input items that Python normalizes or removes before replay.""" + EXTRA_BODY = "extra_body" + """Extra body fields that Python transforms and merges into the provider request.""" + PROVIDER_AFFINITY = "provider_affinity" + """A provider affinity header that Python validates and populates from the session id.""" + PRE_REQUEST_HOOK = "pre_request_hook" + """A logger overriding async_pre_request_hook, which may rewrite the request.""" + WEBSEARCH_INTERCEPTION = "websearch_interception" + """A WebSearchInterceptionLogger that may answer without calling the provider.""" + MESSAGES_INTERCEPTOR = "messages_interceptor" + """A registered MessagesInterceptor that takes over the call.""" + AGENTIC_LOOP_HOOK = "agentic_loop_hook" + """A logger overriding async_should_run_agentic_loop, which may run follow-up calls.""" + RESPONSE_CACHE = "response_cache" + """litellm.cache, whose hit and store order differs from native.""" + TLS_POLICY = "tls_policy" + """An OpenSSL security level or ECDH curve that native TLS cannot apply.""" + MANAGED_RESOURCE_IDS = "managed_resource_ids" + """Managed file or vector store ids that Python maps to provider ids.""" + CACHE_CONTROL_MARKS = "cache_control_marks" + """Anthropic cache_control marks that Python strips or turns into prompt cache breakpoints.""" + ANTHROPIC_TOOL_SEARCH = "anthropic_tool_search" + """Anthropic tool search tools that Python maps to OpenAI tool search.""" + TOOL_SCHEMA_REPAIR = "tool_schema_repair" + """Tool schemas that Python decodes, flattens or strips of non-Python regex patterns.""" + MODEL_PARAM_GATING = "model_param_gating" + """Params that Python clamps, drops or rejects from the model map, like gpt-5 temperature.""" + WORKLOAD_IDENTITY = "workload_identity" + """No static key, so Python mints a workload identity federation token.""" + CACHE_CONTROL_INJECTION = "cache_control_injection" + """Prompt caching injection that Python applies to messages and system.""" + STREAM_COST_INJECTION = "stream_cost_injection" + """litellm.include_cost_in_streaming_usage, where Python adds cost to streamed usage.""" + THINKING_SIGNATURE_RETRY = "thinking_signature_retry" + """Replayed thinking blocks, which Python strips and retries once if Anthropic rejects their signature.""" + CLIENT_HEADERS = "client_headers" + """Proxy-forwarded headers that Python merges into extra_headers.""" + DEPLOYMENT_HOOK = "deployment_hook" + """A logger overriding async_pre_call_deployment_hook, which the sync Python handler runs itself.""" + MODEL_LIST = "model_list" + """A model_list that Python fans out across its deployments.""" + MODEL_ALIAS_MAP = "model_alias_map" + """A model that litellm.model_alias_map rewrites before the call.""" @dataclass(frozen=True, slots=True) @@ -68,14 +139,78 @@ class RouteRule: Rules: TypeAlias = tuple[RouteRule, ...] +COMMON_PYTHON_FEATURES: Final = frozenset( + { + Feature.MOCK_RESPONSE, + Feature.PROXY_MCP_TOOLS, + Feature.CALLER_CLIENT, + Feature.TIMEOUT_VARIANTS, + Feature.AGENTIC_LOOP_HOOK, + Feature.TLS_POLICY, + } +) + +CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { + Feature.RESPONSE_CACHE, + Feature.STREAMING, + Feature.DROP_PARAMS, + Feature.MODIFY_PARAMS, + Feature.EXTRA_BODY, + Feature.RETRIES_FALLBACKS, + Feature.PROMPT_MANAGEMENT, + Feature.CACHE_CONTROL_INJECTION, + Feature.MODEL_PARAM_GATING, + Feature.MODEL_LIST, + Feature.MODEL_ALIAS_MAP, +} + +MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { + Feature.SAFEGUARDS_CLASSIFIER, + Feature.PRE_REQUEST_HOOK, + Feature.WEBSEARCH_INTERCEPTION, + Feature.MESSAGES_INTERCEPTOR, + Feature.RESPONSE_CACHE, + Feature.MODIFY_PARAMS, + Feature.WORKLOAD_IDENTITY, + Feature.CACHE_CONTROL_INJECTION, + Feature.STREAM_COST_INJECTION, + Feature.THINKING_SIGNATURE_RETRY, +} + +RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { + Feature.DROP_PARAMS, + Feature.MODIFY_PARAMS, + Feature.STREAMING, + Feature.PROMPT_MANAGEMENT, + Feature.TEXT_FORMAT, + Feature.CHAT_COMPLETIONS_BRIDGE, + Feature.RETRIES_FALLBACKS, + Feature.DEPLOYMENT_RESPONSE_ID, + Feature.ENCODED_ITEM_IDS, + Feature.INPUT_REWRITE, + Feature.EXTRA_BODY, + Feature.PROVIDER_AFFINITY, + Feature.RESPONSE_CACHE, + Feature.MANAGED_RESOURCE_IDS, + Feature.CACHE_CONTROL_MARKS, + Feature.ANTHROPIC_TOOL_SEARCH, + Feature.TOOL_SCHEMA_REPAIR, + Feature.MODEL_PARAM_GATING, + Feature.WORKLOAD_IDENTITY, + Feature.CLIENT_HEADERS, + Feature.DEPLOYMENT_HOOK, +} + RULES: Final[Rules] = ( + RouteRule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY, features=CHAT_PYTHON_FEATURES), RouteRule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY), RouteRule(Route.EMBEDDINGS, Rollout.PYTHON_ONLY), RouteRule(Route.OCR, Rollout.RUST_REQUIRED), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, targets=frozenset({Target.CHAT_COMPLETIONS, Target.RESPONSES})), - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.SAFEGUARDS_CLASSIFIER})), + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=MESSAGES_PYTHON_FEATURES), RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN, providers=frozenset({"anthropic"})), RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY), + RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY, features=RESPONSES_PYTHON_FEATURES), RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY), RouteRule(Route.TOKEN_COUNTER, Rollout.PYTHON_ONLY), RouteRule(Route.TOKENIZER, Rollout.PYTHON_ONLY), diff --git a/litellm/rust_bridge/chat_completions/detectors.py b/litellm/rust_bridge/chat_completions/detectors.py new file mode 100644 index 00000000000..5aa65a945b1 --- /dev/null +++ b/litellm/rust_bridge/chat_completions/detectors.py @@ -0,0 +1,137 @@ +"""Detectors for Python-only Chat Completions behavior.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter + +import litellm +from litellm.rust_bridge.detectors import ( + agentic_loop_hook, + api_base, + call_callbacks, + mock_response, + overridden, + present, + retries_fallbacks, +) +from litellm.rust_bridge.public_call import NativeCall, optional_sequence, optional_str + +_PARAMS: Final = TypeAdapter(dict[str, object]) + + +def chat_mock_response(request: NativeCall) -> bool: + """Chat mocks can supply text, tool calls or a timeout without provider IO.""" + return ( + mock_response(request) + or bool(request.resolved.get("mock_tool_calls")) + or bool(request.resolved.get("mock_timeout")) + ) + + +def model_list(request: NativeCall) -> bool: + """Python fans the request out across the supplied model deployments.""" + return present(request, "model_list") + + +def model_alias_map(request: NativeCall) -> bool: + """Python rewrites this model through the configured alias map.""" + return bool(litellm.model_alias_map) and str(request.resolved["model"]) in litellm.model_alias_map + + +def chat_retries_fallbacks(request: NativeCall) -> bool: + """Chat also reads global model fallbacks and context-window fallbacks.""" + return ( + retries_fallbacks(request) + or bool(optional_sequence(litellm.model_fallbacks)) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm annotates it with a bare list + or present(request, "context_window_fallback_dict") + ) + + +_CHAT_UNGATED: Final = frozenset( + { + "model", + "messages", + "custom_llm_provider", + "api_version", + "drop_params", + "allowed_openai_params", + "additional_drop_params", + } +) + + +_NAMES: Final = TypeAdapter(list[str]) + + +def _chat_non_default_params(request: NativeCall) -> Mapping[str, object]: + from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES + + defaults: Final = _PARAMS.validate_python(DEFAULT_CHAT_COMPLETION_PARAM_VALUES) + return { + name: value + for name, value in request.resolved.items() + if name in defaults and name not in _CHAT_UNGATED and value != defaults[name] + } + + +def _chat_provider(request: NativeCall) -> tuple[str, str | None]: + from litellm.exceptions import BadRequestError + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model: Final = str(request.resolved["model"]) + provider: Final = optional_str(request.resolved.get("custom_llm_provider")) + try: + resolved_model, resolved_provider, _, _ = get_llm_provider(model, provider, api_base=api_base(request)) + except BadRequestError: + return model, provider + return resolved_model, resolved_provider + + +def _unvalidated(params: Mapping[str, object]) -> frozenset[str]: + from litellm.utils import PROVIDER_UNVALIDATED_PARAMS + + return PROVIDER_UNVALIDATED_PARAMS | (frozenset({"n"}) if params.get("n") == 1 else frozenset[str]()) + + +def chat_model_param_gating(request: NativeCall) -> bool: + """Python rejects unsupported chat params or maps them differently with drop_params.""" + from litellm.exceptions import UnsupportedParamsError + from litellm.litellm_core_utils.get_supported_openai_params import ( + get_supported_openai_params, # pyright: ignore[reportUnknownVariableType] # returns a bare list + ) + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + params: Final = _chat_non_default_params(request) + model, provider = _chat_provider(request) + if not params or provider is None: + return False + supported: Final = frozenset(_NAMES.validate_python(get_supported_openai_params(model, provider) or [])) + if any(name not in supported for name in params.keys() - _unvalidated(params)): + return True + try: + llm_provider: Final = LlmProviders(provider) + except ValueError: + return False + config: Final = ProviderConfigManager.get_provider_chat_config(model, llm_provider) + if config is None: + return False + try: + strict: Final = _PARAMS.validate_python( + config.map_openai_params(dict(params), {}, model, drop_params=False) # pyright: ignore[reportUnknownMemberType] # returns a bare dict + ) + except UnsupportedParamsError: + return True + return strict != _PARAMS.validate_python( + config.map_openai_params(dict(params), {}, model, drop_params=True) # pyright: ignore[reportUnknownMemberType] # returns a bare dict + ) + + +def chat_agentic_loop_hook(request: NativeCall) -> bool: + """Chat callbacks can also override the chat-completion-specific agentic loop.""" + return agentic_loop_hook(request) or overridden( + "async_should_run_chat_completion_agentic_loop", call_callbacks(request) + ) diff --git a/litellm/rust_bridge/detectors.py b/litellm/rust_bridge/detectors.py new file mode 100644 index 00000000000..0ad1b787e38 --- /dev/null +++ b/litellm/rust_bridge/detectors.py @@ -0,0 +1,183 @@ +"""Shared feature detectors and helpers for native-route selection.""" + +from __future__ import annotations + +import os +from collections.abc import Callable, Iterable, Mapping +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeAlias, cast # noqa: TID251 # tool lists stay caller-owned + +import httpx + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.rust_bridge.catalog import Feature +from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str + +if TYPE_CHECKING: + from litellm.responses.mcp.litellm_proxy_mcp_handler import ToolParam + +Detector: TypeAlias = Callable[[NativeCall], bool] +Detectors: TypeAlias = Mapping[Feature, Detector] + + +def detected(detectors: Detectors, request: NativeCall) -> frozenset[Feature]: + return frozenset(feature for feature, detector in detectors.items() if detector(request)) + + +def present(request: NativeCall, name: str) -> bool: + return request.resolved.get(name) is not None + + +def mock_response(request: NativeCall) -> bool: + """Python returns a supplied mock instead of calling the provider.""" + return present(request, "mock_response") + + +def caller_client(request: NativeCall) -> bool: + """Python sends through the caller's HTTP client or shared session.""" + return present(request, "client") or present(request, "shared_session") + + +def extra_body(request: NativeCall) -> bool: + """Python transforms and merges additional fields into the provider body.""" + return bool(request.resolved.get("extra_body")) + + +def streaming(request: NativeCall) -> bool: + """The caller requested streaming, which some native routes do not support yet.""" + return request.resolved.get("stream") is True + + +def timeout_variants(request: NativeCall) -> bool: + """Python handles request and stream overrides and httpx phase timeouts.""" + return ( + present(request, "request_timeout") + or (streaming(request) and present(request, "stream_timeout")) + or isinstance(request.resolved.get("timeout"), httpx.Timeout) + ) + + +def proxy_mcp_tools(request: NativeCall) -> bool: + """Python expands proxy MCP tools and may execute their tool calls.""" + from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler + + tools: Final = optional_sequence(request.resolved.get("tools")) + return tools is not None and LiteLLM_Proxy_MCP_Handler.should_use_litellm_mcp_gateway( + cast("Iterable[ToolParam]", tools) # cast-ok: the predicate skips entries that are not MCP tool dicts + ) + + +def prompt_management(request: NativeCall) -> bool: + """Python resolves prompt-manager inputs or strips the litellm_agent prefix.""" + from litellm.integrations.litellm_agent.litellm_agent_model_resolver import LITELLM_AGENT_PREFIX + from litellm.types.utils import DynamicPromptManagementParamLiteral + + return ( + present(request, "prompt_id") + or str(request.resolved["model"]).startswith(LITELLM_AGENT_PREFIX) + or any(request.resolved.get(param) for param in DynamicPromptManagementParamLiteral.list_all_params()) + ) + + +def retries_fallbacks(request: NativeCall) -> bool: + """Python runs the requested retry policy or fallback chain.""" + return bool(litellm.num_retries) or any( + present(request, name) for name in ("num_retries", "max_retries", "fallbacks", "retry_policy") + ) + + +def api_base(request: NativeCall) -> str | None: + return optional_str(request.resolved.get("api_base")) or optional_str(request.resolved.get("base_url")) + + +def cache_control_injection(request: NativeCall) -> bool: + """Python injects Anthropic prompt-cache breakpoints into messages and system content.""" + from litellm.integrations.anthropic_cache_control_hook import configured_injection_points + + return ( + litellm.enable_anthropic_prompt_caching is True + or request.resolved.get("enable_prompt_caching") is True + or bool(configured_injection_points(request.resolved.get("cache_control_injection_points"))) + ) + + +def drop_params(request: NativeCall) -> bool: + """Python drops unsupported or explicitly excluded request parameters.""" + return ( + bool(litellm.drop_params) + or normalize_drop_params(request.resolved.get("drop_params")) is True + or bool(optional_sequence(request.resolved.get("additional_drop_params"))) + ) + + +def modify_params(_request: NativeCall) -> bool: + """Python rewrites unsupported request values when modify_params is enabled.""" + return bool(litellm.modify_params) + + +def _custom_loggers(callbacks: Iterable[object]) -> tuple[CustomLogger, ...]: + from litellm.litellm_core_utils.litellm_logging import get_custom_logger_compatible_class + + resolved: Final = ( + get_custom_logger_compatible_class(callback) if isinstance(callback, str) else callback + for callback in callbacks + ) + return tuple(callback for callback in resolved if isinstance(callback, CustomLogger)) + + +def overridden(method: str, callbacks: Iterable[object]) -> bool: + base: Final = getattr(CustomLogger, method) + return any(getattr(type(callback), method, base) is not base for callback in _custom_loggers(callbacks)) + + +def _request_success_callbacks(request: NativeCall) -> tuple[object, ...]: + from litellm.litellm_core_utils.litellm_logging import Logging + + supplied: Final = optional_sequence(request.resolved.get("success_callback")) or () + logging_obj: Final = request.resolved.get("litellm_logging_obj") + dynamic: Final = ( + optional_sequence(logging_obj.dynamic_success_callbacks) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # Logging annotates the list with a bare Callable + if isinstance(logging_obj, Logging) + else None + ) + return (*supplied, *(dynamic or ())) + + +def call_callbacks(request: NativeCall) -> tuple[object, ...]: + return (*litellm.callbacks, *_request_success_callbacks(request)) + + +def agentic_loop_hook(request: NativeCall) -> bool: + """A callback can request follow-up tool calls through Python's agentic loop.""" + return overridden("async_should_run_agentic_loop", call_callbacks(request)) + + +def response_cache(request: NativeCall) -> bool: + """Python owns cache lookup and storage unless this call disables cache reads.""" + cache: Final = optional_mapping(request.resolved.get("cache")) or {} + return litellm.cache is not None and cache.get("no-cache") is not True + + +_NATIVE_ECDH_CURVES: Final = frozenset({"x25519", "prime256v1", "secp256r1", "p-256", "secp384r1", "p-384"}) + + +def tls_policy(_request: NativeCall) -> bool: + """Python applies TLS settings or ECDH curves the native client cannot configure.""" + curve: Final = os.getenv("SSL_ECDH_CURVE", litellm.ssl_ecdh_curve) + return bool(os.getenv("SSL_SECURITY_LEVEL", litellm.ssl_security_level)) or ( + bool(curve) and str(curve).strip().lower() not in _NATIVE_ECDH_CURVES + ) + + +COMMON: Final[Detectors] = MappingProxyType( + { + Feature.MOCK_RESPONSE: mock_response, + Feature.PROXY_MCP_TOOLS: proxy_mcp_tools, + Feature.CALLER_CLIENT: caller_client, + Feature.TIMEOUT_VARIANTS: timeout_variants, + Feature.AGENTIC_LOOP_HOOK: agentic_loop_hook, + Feature.TLS_POLICY: tls_policy, + } +) diff --git a/litellm/rust_bridge/dispatch.py b/litellm/rust_bridge/dispatch.py index bfc449ca3bc..ca98fe2b5b2 100644 --- a/litellm/rust_bridge/dispatch.py +++ b/litellm/rust_bridge/dispatch.py @@ -4,6 +4,7 @@ from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from typing import Final, Generic, TypeAlias, TypeVar +from litellm._logging import verbose_logger from litellm.rust_bridge import catalog, runtime from litellm.rust_bridge.bindings import NativeBinding from litellm.rust_bridge.catalog import Route, RouteContext, Rules @@ -38,6 +39,15 @@ class PublicDispatch(Generic[RequestT]): ) return request + def _guarded_context(self, request: RequestT) -> RouteContext | None: + try: + return self.context(request) + except Exception: # noqa: BLE001 # detectors call legacy handlers, so any error must fall back to Python + verbose_logger.warning( + "%s route detection failed, serving the call on Python", self.route.value, exc_info=True + ) + return None + def run( self, args: tuple[object, ...], @@ -63,8 +73,11 @@ class PublicDispatch(Generic[RequestT]): request: Final = self.request(args, kwargs) if request is None: return python(*args, **kwargs) + context: Final = self._guarded_context(request) + if context is None: + return python(*args, **kwargs) return runtime.run( - self.context(request), + context, binding=binding, native=lambda hook: native(hook, request, args, kwargs), python=lambda: python(*args, **kwargs), @@ -96,8 +109,11 @@ class PublicDispatch(Generic[RequestT]): request: Final = self.request(args, kwargs) if request is None: return await python(*args, **kwargs) + context: Final = self._guarded_context(request) + if context is None: + return await python(*args, **kwargs) return await runtime.arun( - self.context(request), + context, binding=binding, native=lambda hook: native(hook, request, args, kwargs), python=lambda: python(*args, **kwargs), diff --git a/litellm/rust_bridge/host/routes/responses.py b/litellm/rust_bridge/host/routes/responses.py index 470a89a0e05..22e8cbbf3b0 100644 --- a/litellm/rust_bridge/host/routes/responses.py +++ b/litellm/rust_bridge/host/routes/responses.py @@ -4,9 +4,8 @@ from collections.abc import Mapping from typing import Final import litellm -from litellm import get_llm_provider from litellm.rust_bridge import failures -from litellm.rust_bridge.public_call import inference_decline_reason, known_params, optional_str +from litellm.rust_bridge.public_call import known_params, optional_str from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__) @@ -34,14 +33,3 @@ def map_failure(error: Exception, request: Mapping[str, object]) -> Exception: request, optional_str(request.get("api_base")) or optional_str(request.get("base_url")), ) - - -def decline_reason(request: Mapping[str, object]) -> str | None: - if optional_str(request.get("custom_llm_provider")) is None and "/" not in str(request["model"]): - try: - _, provider, _, _ = get_llm_provider(model=str(request["model"])) - except litellm.exceptions.BadRequestError: - return "native Responses could not resolve the provider" - if provider != "openai": - return "native HTTP responses provider" - return inference_decline_reason(PARAMETERS, request) diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py new file mode 100644 index 00000000000..691c04f2daa --- /dev/null +++ b/litellm/rust_bridge/messages/detectors.py @@ -0,0 +1,85 @@ +"""Detectors for Python-only Anthropic Messages behavior.""" + +from __future__ import annotations + +from typing import Final + +import litellm +from litellm.rust_bridge.detectors import overridden, present, streaming +from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str + + +def anthropic_workload_identity(request: NativeCall) -> bool: + """Python mints an Anthropic federation token when no static credential is configured.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + from litellm.secret_managers.main import get_secret_str + + def configured(param: str, env: str) -> bool: + return present(request, param) or get_secret_str(env) is not None + + return ( + AnthropicModelInfo.get_api_key(optional_str(request.resolved.get("api_key"))) is None + and AnthropicModelInfo.get_auth_token() is None + and configured("anthropic_federation_rule_id", "ANTHROPIC_FEDERATION_RULE_ID") + and configured("anthropic_organization_id", "ANTHROPIC_ORGANIZATION_ID") + ) + + +_THINKING_BLOCKS: Final = frozenset({"thinking", "redacted_thinking"}) + + +def _replays_thinking(message: object) -> bool: + content: Final = optional_sequence((optional_mapping(message) or {}).get("content")) or () + return any((optional_mapping(block) or {}).get("type") in _THINKING_BLOCKS for block in content) + + +def thinking_signature_retry(request: NativeCall) -> bool: + """Python may strip replayed thinking blocks and retry after a rejected signature.""" + return any(_replays_thinking(message) for message in optional_sequence(request.resolved.get("messages")) or ()) + + +def stream_cost_injection(request: NativeCall) -> bool: + """Python adds response cost to streaming usage when the global setting is truthy.""" + return streaming(request) and bool(litellm.include_cost_in_streaming_usage) + + +def pre_request_hook(_request: NativeCall) -> bool: + """The async Messages handler runs callbacks that can rewrite the request.""" + return overridden("async_pre_request_hook", litellm.callbacks) + + +def websearch_interception(_request: NativeCall) -> bool: + """The async Messages handler can answer a web-search request without an LLM call.""" + from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger + + return any(isinstance(callback, WebSearchInterceptionLogger) for callback in litellm.callbacks) + + +def _resolved_provider(request: NativeCall) -> str | None: + from litellm.exceptions import BadRequestError + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + provider: Final = optional_str(request.resolved.get("custom_llm_provider")) + try: + return get_llm_provider(str(request.resolved["model"]), provider)[1] + except BadRequestError: + return provider + + +def intercepted(request: NativeCall) -> bool: + """The async Messages handler delegates matching tools to a registered interceptor.""" + from litellm.llms.anthropic.pass_through.messages.interceptors import get_messages_interceptors + + tools: Final = optional_sequence(request.resolved.get("tools")) + if not tools: + return False + mappings: Final = tuple(mapping for mapping in map(optional_mapping, tools) if mapping is not None) + provider: Final = _resolved_provider(request) + return any(interceptor.can_handle(mappings, provider) for interceptor in get_messages_interceptors()) + + +def safeguards_classifier(request: NativeCall) -> bool: + """Python's proxy classifier evaluates dangerous tool use for non-Claude models.""" + from litellm.llms.anthropic.pass_through.safeguards import native_route_classifies + + return native_route_classifies(str(request.resolved["model"]), request.resolved.get("safeguards")) diff --git a/litellm/rust_bridge/public_call.py b/litellm/rust_bridge/public_call.py index 4b69074d057..8625ca71bb2 100644 --- a/litellm/rust_bridge/public_call.py +++ b/litellm/rust_bridge/public_call.py @@ -8,33 +8,6 @@ from dataclasses import dataclass from types import MappingProxyType from typing import Final, TypeVar, cast # noqa: TID251 # narrows caller-owned containers without copying them -import litellm - -_INFERENCE_CONTEXT: Final = frozenset( - { - "model", - "messages", - "input", - "api_key", - "api_base", - "base_url", - "custom_llm_provider", - "extra_headers", - "timeout", - "request_timeout", - "callbacks", - "success_callback", - "failure_callback", - "metadata", - "litellm_metadata", - "litellm_call_id", - "litellm_trace_id", - "litellm_logging_obj", - "litellm_credential_name", - "proxy_server_request", - } -) - def signature(legacy: Callable[..., object]) -> inspect.Signature: return inspect.signature(legacy) @@ -75,19 +48,6 @@ def known_params(names: Collection[str], request: Mapping[str, object]) -> Mappi return {name: value for name, value in request.items() if name in names and value is not None} -def inference_decline_reason(parameters: tuple[str, ...], kwargs: Mapping[str, object]) -> str | None: - if litellm.drop_params or litellm.modify_params: - return "native inference does not implement the configured parameter rewrites" - for name, value in kwargs.items(): - if value is None: - continue - if name in {"cache", "caching"}: - continue - if name not in parameters and name not in _INFERENCE_CONTEXT: - return f"native inference does not implement {name}" - return None - - _BAGS: Final = frozenset({inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD}) diff --git a/litellm/rust_bridge/responses/detectors.py b/litellm/rust_bridge/responses/detectors.py new file mode 100644 index 00000000000..392c11f48a8 --- /dev/null +++ b/litellm/rust_bridge/responses/detectors.py @@ -0,0 +1,210 @@ +"""Detectors for Python-only Responses behavior.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from itertools import accumulate, chain, repeat, takewhile +from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # tool lists stay caller-owned + +from pydantic import TypeAdapter + +import litellm +from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present +from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str + +if TYPE_CHECKING: + from openai.types.responses import ResponseInputParam + + from litellm.types.llms.openai import ALL_RESPONSES_API_TOOL_PARAMS, ResponsesAPIOptionalRequestParams + + +def client_headers(request: NativeCall) -> bool: + """Python merges proxy-forwarded headers into extra_headers before sending.""" + return bool(optional_mapping(request.resolved.get("headers"))) + + +def provider_affinity(request: NativeCall) -> bool: + """Python validates and fills the affinity header from the session ID.""" + return present(request, "provider_affinity_header") + + +def text_format(request: NativeCall) -> bool: + """Python builds the output schema and parses the response into the supplied type.""" + return present(request, "text_format") + + +def chat_completions_bridge(request: NativeCall) -> bool: + """Python serves this Responses request through Chat Completions.""" + from litellm.responses.main import ( + _normalize_openai_chat_completions_responses_model, # pyright: ignore[reportPrivateUsage] # the handler's own bridge prefix check + ) + + return ( + bool(request.resolved.get("use_chat_completions_api")) + or _normalize_openai_chat_completions_responses_model(str(request.resolved["model"]))[1] + ) + + +def _openai_model(request: NativeCall) -> str: + from litellm.exceptions import BadRequestError + from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider + + model: Final = str(request.resolved["model"]) + try: + return get_llm_provider(model, optional_str(request.resolved.get("custom_llm_provider")))[0] + except BadRequestError: + return model + + +def _model_info(request: NativeCall) -> Mapping[str, object] | None: + return optional_mapping(request.resolved.get("model_info")) + + +def _tools(request: NativeCall) -> Sequence[object]: + return optional_sequence(request.resolved.get("tools")) or () + + +def managed_resource_ids(request: NativeCall) -> bool: + """Python replaces managed file IDs in Responses input and tools with provider IDs.""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + update_responses_input_with_model_file_ids, + update_responses_tools_with_model_file_ids, + ) + + model_id: Final = optional_str((_model_info(request) or {}).get("id")) + mapping: Final = request.resolved.get("model_file_id_mapping") + file_ids: Final = cast("dict[str, dict[str, str]] | None", mapping) # cast-ok: handler passes it unvalidated + input_value: Final = request.resolved.get("input") + tools: Final = [dict(tool) for tool in map(optional_mapping, _tools(request)) if tool is not None] + rewritten_tools: Final = update_responses_tools_with_model_file_ids(tools, model_id, file_ids) if tools else tools + return update_responses_input_with_model_file_ids(input_value, model_id, file_ids) != input_value or ( + rewritten_tools != tools + ) + + +def input_rewrite(request: NativeCall) -> bool: + """Python normalizes replayed Responses items and removes unsupported IDs or namespaces.""" + from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + + input_value: Final = request.resolved.get("input") + if not isinstance(input_value, list): + return False + config: Final = OpenAIResponsesAPIConfig() + replayable: Final = config._drop_bridge_minted_reasoning_items( # pyright: ignore[reportPrivateUsage] # use the handler's replay policy + cast("ResponseInputParam", input_value), # cast-ok: the handler accepts unvalidated input items + ) + normalized: Final = config._validate_input_param(replayable) # pyright: ignore[reportPrivateUsage] # use the handler's normalization + cleaned: Final = config._drop_foreign_tool_call_item_ids(normalized) # pyright: ignore[reportPrivateUsage] # use the handler's id cleanup + return config.strip_custom_tool_call_namespace_from_responses_input(cleaned) != input_value + + +def _children(value: object) -> tuple[object, ...]: + if isinstance(value, Mapping): + return tuple(cast("Mapping[object, object]", value).values()) # cast-ok: only values are walked + if isinstance(value, list | tuple): + return tuple(cast("Sequence[object]", value)) # cast-ok: only items are walked + return () + + +def _next_level(level: tuple[object, ...], _: None) -> tuple[object, ...]: + return tuple(chain.from_iterable(map(_children, level))) + + +def _mentions(value: object, key: str) -> bool: + levels: Final = takewhile(bool, accumulate(repeat(None), _next_level, initial=(value,))) + return any( + key in cast("Mapping[object, object]", node) # cast-ok: node passed the Mapping guard + for node in chain.from_iterable(levels) + if isinstance(node, Mapping) + ) + + +def cache_control_marks(request: NativeCall) -> bool: + """Python strips or translates cache_control marks in Responses input and tools.""" + return _mentions(request.resolved.get("input"), "cache_control") or _mentions(_tools(request), "cache_control") + + +def anthropic_tool_search(request: NativeCall) -> bool: + """Python translates Anthropic tool-search definitions for the Responses API.""" + from litellm.responses.litellm_completion_transformation.tool_search import is_anthropic_tool_search_chat_tool + + return any(is_anthropic_tool_search_chat_tool(tool) for tool in _tools(request)) + + +def _undecoded_schema(tool: object) -> bool: + mapping: Final = optional_mapping(tool) + parameters: Final = None if mapping is None else mapping.get("parameters") + return parameters is not None and not isinstance(parameters, Mapping) + + +def tool_schema_repair(request: NativeCall) -> bool: + """Python decodes or repairs tool schemas before sending them to OpenAI.""" + from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + from litellm.types.router import GenericLiteLLMParams + + tools: Final = _tools(request) + if any(_undecoded_schema(tool) for tool in tools): + return True + sanitized: Final = OpenAIResponsesAPIConfig()._sanitized_tool_schemas_for_openai( # pyright: ignore[reportPrivateUsage] # the handler's own schema rewrite + model=_openai_model(request), + tools=cast("Sequence[ALL_RESPONSES_API_TOOL_PARAMS]", tools), # cast-ok: sanitizer skips non-dict tools + litellm_params=GenericLiteLLMParams(model_info=_model_info(request)), + ) + return bool(tools) and sanitized is not tools + + +_PARAMS: Final = TypeAdapter(dict[str, object]) + + +def model_param_gating(request: NativeCall) -> bool: + """Python clamps, drops or remaps Responses params according to the model's capabilities.""" + from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig + from litellm.rust_bridge.host.routes.responses import PARAMETERS + + params: Final = {name: request.resolved[name] for name in PARAMETERS if request.resolved.get(name) is not None} + mapped: Final = _PARAMS.validate_python( + OpenAIResponsesAPIConfig().map_openai_params( # pyright: ignore[reportUnknownMemberType] # returns a bare dict + cast("ResponsesAPIOptionalRequestParams", params), # cast-ok: the handler passes these kwargs + _openai_model(request), + drop_params=True, + ) + ) + return mapped != params + + +def openai_workload_identity(request: NativeCall) -> bool: + """Python mints an OpenAI federation token when no static API key is configured.""" + from litellm.llms.openai.workload_identity import resolve_openai_workload_identity_config + + api_key: Final = optional_str(request.resolved.get("api_key")) or litellm.api_key or litellm.openai_key + return resolve_openai_workload_identity_config(api_key, api_base(request), request.resolved) is not None + + +def deployment_response_id(request: NativeCall) -> bool: + """Python encodes the deployment ID into the returned Responses ID.""" + metadata: Final = optional_mapping(request.resolved.get("litellm_metadata")) or {} + model_info: Final = optional_mapping(metadata.get("model_info")) or {} + return model_info.get("id") is not None + + +def _encoded_item(item: Mapping[str, object]) -> bool: + from litellm.responses.utils import ResponsesAPIRequestUtils + + item_id: Final = optional_str(item.get("id")) + encrypted_content: Final = optional_str(item.get("encrypted_content")) + return (item_id is not None and ResponsesAPIRequestUtils.decode_encrypted_item_id(item_id) is not None) or ( + encrypted_content is not None + and ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id(encrypted_content)[1] != encrypted_content + ) + + +def encoded_item_ids(request: NativeCall) -> bool: + """Python restores provider IDs or encrypted content from previously wrapped items.""" + items: Final = optional_sequence(request.resolved.get("input")) or () + mappings: Final = (optional_mapping(item) for item in items) + return any(_encoded_item(item) for item in mappings if item is not None) + + +def deployment_hook(request: NativeCall) -> bool: + """The sync Responses handler runs deployment hooks that native currently skips.""" + return overridden("async_pre_call_deployment_hook", call_callbacks(request)) diff --git a/tests/test_litellm_rust/AGENTS.md b/tests/test_litellm_rust/AGENTS.md index e115819be9d..f688e2253e8 100644 --- a/tests/test_litellm_rust/AGENTS.md +++ b/tests/test_litellm_rust/AGENTS.md @@ -1,33 +1,62 @@ -# Invariants +# Scope -- Outside the dispatch tests below, this directory tests the Rust path and nothing else. Treat the Python implementation of every route as already removed. Never call it on purpose, never run one request through both engines, and never compare Python and Rust outputs, headers, cache keys or logs -- A test that would still pass with Python serving the call does not belong here. Every test except the dispatch tests asserts the call was served natively with `support.routing.assert_served_natively` -- Dispatch tests check which path a public call takes under given rules. They are each route folder's `test_dispatch.py`, plus `test_dispatch.py` and `test_public_entries.py` at the top of this directory - - They pass `rules=` to a dispatch directly or wrap the whole test in one `support.routing.routed(...)` block, and they stub the native bindings - - They may call Python, since the choice between the two paths is what they test - - They live here until they can run in `tests/unit/` without the compiled extension -- Select the Rust path with `support.routing.native_route(...)`, which prepends `RUST_REQUIRED` rules to `catalog.RULES` and restores them on exit. Each route folder's `conftest.py` applies it to every test through an autouse fixture. A test outside a route folder wraps its whole body in one `native_route(...)` block, and a child interpreter script assigns `catalog.RULES` once at its top. Never select a backend by reading or setting `LITELLM_RUST`, calling `litellm.rust(...)` or rebinding `catalog.RULES` any other way. The rollout in `litellm/rust_bridge/catalog.py` is a product decision, and a test here must pass no matter what it says, including for a route marked `PYTHON_ONLY` -- `catalog.RULES` is static at runtime and `native_route(...)` swaps a module global, so open it once per test and never per call. Two overlapping blocks, such as one per task inside an `asyncio.gather`, restore out of order and leave the wrong rules behind for later tests -- No test here depends on the rollout switch. Only no rule, `PYTHON_ONLY` and `RUST_REQUIRED` pick a path without it, so those are the only rollouts a test here uses, and Rust with Python fallback is not reachable from this directory. What `LITELLM_RUST` and `litellm.rust(...)` mean is tested in `tests/unit/rust_bridge/test_configuration.py`, mostly through `decide`, which takes both as plain values -- A test lives here only when it cannot be written in Rust. That means it needs the Python public API, Python-owned state such as `litellm.cache` and callbacks, or the process boundary (fork, threads, event loops). Anything checkable from the Rust side goes in the crate's own tests +- This directory tests the Rust path end to end through the Python public API, plus the dispatch tests below, and nothing else +- A test belongs here only when it cannot be written in Rust, meaning it needs + - the Python public API + - Python-owned state such as `litellm.cache` and callbacks + - the process boundary (fork, threads, event loops) +- Anything checkable from the Rust side goes in the crate's own tests - The wire is the local recording server. No test here reaches a real provider -- Never parametrize one test across routes. Each route folder owns its tests, its `server` fixture, its request data and its `call`. The one exception is `test_public_entries.py`, whose subject is that every route serves its public call forms the same way -- Call `_native` directly only for a case the public API cannot reach + +# Rust only + +- Outside the dispatch tests, treat every route's Python implementation as already removed + - never call it on purpose + - never run one request through both engines + - never compare Python and Rust outputs, headers, cache keys or logs +- A test that would still pass with Python serving the call does not belong here +- Every test except the dispatch tests asserts native serving with `support.routing.assert_served_natively` + +# Dispatch tests + +- They check which path a public call takes under given rules + - each route folder's `test_dispatch.py` + - `test_dispatch.py` at the top, which tests `PublicDispatch` with stub routes + - `test_public_entries.py` at the top, which runs every route's public call forms through one table +- They pass `rules=` to a dispatch directly or wrap the whole test in one `support.routing.routed(...)` block, and they stub the native bindings +- They may call Python, since the choice between the two paths is what they test +- They live here until they can run in `tests/unit/` without the compiled extension + +# Selecting a path + +- Use `support.routing.native_route(...)`, which prepends `RUST_REQUIRED` rules to `catalog.RULES` and restores them on exit + - route folders apply it to every test through the autouse fixture in their `conftest.py` + - a test outside a route folder wraps its whole body in one `native_route(...)` block + - a child interpreter script assigns `catalog.RULES` once at its top +- Dispatch tests use `support.routing.routed(...)` instead, which sets `catalog.RULES` to exactly the rules given +- Open either once per test, never per call. They swap a module global, so overlapping blocks (one per task in an `asyncio.gather`) restore out of order and leak rules into later tests +- Never select a backend through `LITELLM_RUST`, `litellm.rust(...)` or any other rebinding of `catalog.RULES` +- No test here depends on the rollout switch + - only no rule, `PYTHON_ONLY` and `RUST_REQUIRED` pick a path without it, so those are the only rollouts used here + - Rust with Python fallback needs the switch, so it is not reachable from this directory + - what `LITELLM_RUST` and `litellm.rust(...)` mean is tested in `tests/unit/rust_bridge/test_configuration.py`, mostly through `decide`, which takes both as plain values +- Tests pass whatever the shipped rollout in `litellm/rust_bridge/catalog.py` says, including for a `PYTHON_ONLY` route # Layout -Each route folder (`ocr/`, `messages/`, `responses/`, `chat_completions/`) uses the same file names, one concern per file - -- `test_dispatch.py` the call reaches the native route, and which path the rules pick for each public call -- `test_requests.py` Python inputs that reach the wire -- `test_failures.py` errors raised to the caller and rejection before provider IO -- `test_settings.py` globals, env, named credentials and their precedence -- `test_streaming.py` stream relay, early close and stream-time logging -- `test_callbacks.py` the logger hook contract -- `test_body_hooks.py` handler body hooks such as interceptors and agentic loops -- `test_lifecycle.py` cancellation, concurrency, context and nested calls -- `test_cache.py` the configured cache seen from this route - -`embeddings/` holds only `test_dispatch.py` so far. At the top, `test_dispatch.py` tests `PublicDispatch` with stub routes and `test_public_entries.py` runs every route's public call forms through one table - -Route data and the route's `call` live in `/data.py`, and `/conftest.py` owns the `server` fixture and the autouse `native` fixture. `support/` holds only route-neutral helpers +- One folder per route (`ocr/`, `messages/`, `responses/`, `chat_completions/`), all using the same file names + - `test_dispatch.py` which path the rules pick for each public call, and that the call reaches the native route + - `test_requests.py` Python inputs that reach the wire + - `test_failures.py` errors raised to the caller and rejection before provider IO + - `test_settings.py` globals, env, named credentials and their precedence + - `test_streaming.py` stream relay, early close and stream-time logging + - `test_callbacks.py` the logger hook contract + - `test_body_hooks.py` handler body hooks such as interceptors and agentic loops + - `test_lifecycle.py` cancellation, concurrency, context and nested calls + - `test_cache.py` the configured cache seen from this route +- `embeddings/` holds only `test_dispatch.py` so far +- `/data.py` holds the route's request data and its `call` +- `/conftest.py` owns the `server` fixture and the autouse `native` fixture +- `support/` holds only route-neutral helpers +- Never parametrize one test across routes, except in `test_public_entries.py`, whose subject is that every route serves its public call forms the same way +- Call `_native` directly only for a case the public API cannot reach diff --git a/tests/test_litellm_rust/chat_completions/test_dispatch.py b/tests/test_litellm_rust/chat_completions/test_dispatch.py index 01249383910..69d9e8e12df 100644 --- a/tests/test_litellm_rust/chat_completions/test_dispatch.py +++ b/tests/test_litellm_rust/chat_completions/test_dispatch.py @@ -4,13 +4,15 @@ from typing import Final, cast # noqa: TID251 # narrows legacy callable signat import pytest import litellm +from litellm.caching.caching import Cache from litellm.chat_completions import dispatch from litellm.chat_completions.dispatch import ( _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch _DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch ) +from litellm.integrations.custom_logger import CustomLogger from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, RouteRule, Rules +from litellm.rust_bridge.catalog import Feature, Route, RouteRule, Rules from litellm.rust_bridge.chat_completions.entrypoints import ( NATIVE_ACOMPLETION, NATIVE_COMPLETION, @@ -228,6 +230,93 @@ async def test_acompletion_marker_binds_positionals_by_the_sync_signature() -> N assert [(request.resolved["timeout"], request.resolved["functions"]) for request in captured] == [(30.0, None)] +class _ChatAgentic(CustomLogger): + async def async_should_run_chat_completion_agentic_loop( + self, *args: object, **kwargs: object + ) -> tuple[bool, dict[str, object]]: # pyright: ignore[reportIncompatibleMethodOverride] # test double only needs the override + return False, {} + + +@pytest.mark.parametrize( + ("feature", "kwargs", "settings"), + ( + (Feature.MOCK_RESPONSE, {"mock_response": "hi"}, {}), + pytest.param(Feature.MOCK_RESPONSE, {"mock_tool_calls": [{"id": "call_1"}]}, {}, id="mock-tool-calls"), + pytest.param(Feature.MOCK_RESPONSE, {"mock_timeout": True}, {}, id="mock-timeout"), + (Feature.CALLER_CLIENT, {"client": object()}, {}), + (Feature.TIMEOUT_VARIANTS, {"request_timeout": 5}, {}), + (Feature.AGENTIC_LOOP_HOOK, {}, {"callbacks": [_ChatAgentic()]}), + (Feature.TLS_POLICY, {}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}), + (Feature.RESPONSE_CACHE, {}, {"cache": Cache()}), + (Feature.STREAMING, {"stream": True}, {}), + (Feature.DROP_PARAMS, {}, {"drop_params": True}), + (Feature.MODIFY_PARAMS, {}, {"modify_params": True}), + (Feature.EXTRA_BODY, {"extra_body": {"top_k": 1}}, {}), + (Feature.RETRIES_FALLBACKS, {"num_retries": 2}, {}), + pytest.param( + Feature.RETRIES_FALLBACKS, + {"context_window_fallback_dict": {"anthropic/claude-sonnet-4-5": "anthropic/claude-opus-4-5"}}, + {}, + id="context-window-fallbacks", + ), + pytest.param( + Feature.RETRIES_FALLBACKS, + {}, + {"model_fallbacks": [{"anthropic/claude-sonnet-4-5": ["anthropic/claude-opus-4-5"]}]}, + id="model-fallbacks", + ), + (Feature.PROMPT_MANAGEMENT, {"prompt_id": "welcome"}, {}), + pytest.param(Feature.PROMPT_MANAGEMENT, {"model": "litellm_agent/gpt-5"}, {}, id="litellm-agent-model"), + (Feature.CACHE_CONTROL_INJECTION, {"enable_prompt_caching": True}, {}), + (Feature.MODEL_PARAM_GATING, {"logit_bias": {"1": 1}}, {}), + ( + Feature.MODEL_LIST, + {"model_list": [{"model_name": "anthropic/claude-sonnet-4-5", "litellm_params": {"model": "gpt-5"}}]}, + {}, + ), + ( + Feature.MODEL_ALIAS_MAP, + {}, + {"model_alias_map": {"anthropic/claude-sonnet-4-5": "anthropic/claude-opus-4-5"}}, + ), + ), +) +def test_each_detected_feature_reaches_its_rule( + monkeypatch: pytest.MonkeyPatch, feature: Feature, kwargs: Mapping[str, object], settings: Mapping[str, object] +) -> None: + rules: Final = ( + RouteRule(Route.CHAT_COMPLETIONS, Rollout.PYTHON_ONLY, features=frozenset({feature})), + RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED), + ) + python_result: Final = ModelResponse() + native_result: Final = ModelResponse() + + def served(call_kwargs: Mapping[str, object]) -> object: + return _DISPATCH.run( + (), + {"model": "anthropic/claude-sonnet-4-5", "messages": MESSAGES, **call_kwargs}, + python=lambda *call_args, **python_kwargs: python_result, + binding=completion_binding(lambda request: native_result), + native=native_call_hook, + rules=rules, + ) + + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "ssl_security_level", None) + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.setattr(litellm, "model_alias_map", {}) + for name in ("num_retries", "model_fallbacks"): + monkeypatch.setattr(litellm, name, None) + for name in ("drop_params", "modify_params", "enable_anthropic_prompt_caching"): + monkeypatch.setattr(litellm, name, False) + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE"): + monkeypatch.delenv(name, raising=False) + assert served({}) is native_result + for name, value in settings.items(): + monkeypatch.setattr(litellm, name, value) + assert served(kwargs) is python_result + + def test_acompletion_flag_hands_off_only_when_true() -> None: expected: Final = ModelResponse() diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 6a82dd20c7e..50f71569d73 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -5,6 +5,9 @@ from typing import Final, cast # noqa: TID251 # narrows legacy callable signat import pytest import litellm +from litellm.caching.caching import Cache +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger from litellm.llms.anthropic.pass_through.messages import handler as python_messages from litellm.messages import dispatch from litellm.messages.dispatch import ( @@ -17,6 +20,7 @@ from litellm.rust_bridge.catalog import Feature, Route, RouteRule, Rules, Target from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES, NativeAmessages, NativeMessages from litellm.rust_bridge.public_call import NativeCall +from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse MESSAGES: Final = [{"role": "user", "content": "hi"}] @@ -425,3 +429,175 @@ def test_messages_binding_error_delegates_unchanged_to_python() -> None: ) assert response is expected + + +class _Rewrite(CustomLogger): + async def async_pre_request_hook(self, model: str, messages: list, kwargs: dict) -> dict | None: # pyright: ignore[reportMissingTypeArgument] # mirrors the CustomLogger signature + return kwargs + + +class _Agentic(CustomLogger): + async def async_should_run_agentic_loop(self, *args: object, **kwargs: object) -> tuple[bool, dict]: # pyright: ignore[reportIncompatibleMethodOverride, reportMissingTypeArgument] # test double only needs the override + return False, {} + + +ADVISOR_TOOL: Final = {"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor", "model": "claude-opus-5-5"} + + +@pytest.mark.parametrize( + ("feature", "model", "kwargs", "settings"), + ( + (Feature.MOCK_RESPONSE, "anthropic/claude-sonnet-4-5", {"mock_response": "hi"}, {}), + ( + Feature.PROXY_MCP_TOOLS, + "anthropic/claude-sonnet-4-5", + {"tools": [{"type": "mcp", "server_label": "gateway", "server_url": "litellm_proxy"}]}, + {}, + ), + (Feature.CALLER_CLIENT, "anthropic/claude-sonnet-4-5", {"client": object()}, {}), + (Feature.TIMEOUT_VARIANTS, "anthropic/claude-sonnet-4-5", {"request_timeout": 5}, {}), + (Feature.AGENTIC_LOOP_HOOK, "anthropic/claude-sonnet-4-5", {}, {"callbacks": [_Agentic()]}), + (Feature.RESPONSE_CACHE, "anthropic/claude-sonnet-4-5", {}, {"cache": Cache()}), + (Feature.TLS_POLICY, "anthropic/claude-sonnet-4-5", {}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}), + (Feature.MODIFY_PARAMS, "anthropic/claude-sonnet-4-5", {}, {"modify_params": True}), + ( + Feature.WORKLOAD_IDENTITY, + "anthropic/claude-sonnet-4-5", + {"anthropic_federation_rule_id": "rule-1", "anthropic_organization_id": "org-1"}, + {}, + ), + (Feature.CACHE_CONTROL_INJECTION, "anthropic/claude-sonnet-4-5", {"enable_prompt_caching": True}, {}), + ( + Feature.THINKING_SIGNATURE_RETRY, + "anthropic/claude-sonnet-4-5", + {"messages": [{"role": "assistant", "content": [{"type": "redacted_thinking", "data": "x"}]}]}, + {}, + ), + ( + Feature.STREAM_COST_INJECTION, + "anthropic/claude-sonnet-4-5", + {"stream": True}, + {"include_cost_in_streaming_usage": True}, + ), + ), +) +def test_each_detected_feature_reaches_its_rule( + monkeypatch: pytest.MonkeyPatch, + feature: Feature, + model: str, + kwargs: Mapping[str, object], + settings: Mapping[str, object], +) -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({feature})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + python_result: Final = response(model) + native_result: Final = response(model) + + def served(call_kwargs: Mapping[str, object]) -> AnthropicMessagesResponse: + messages: Final = call_kwargs.get("messages", MESSAGES) + return _DISPATCH.run( + (16, messages, model), + {name: value for name, value in call_kwargs.items() if name != "messages"}, + python=lambda *call_args, **python_kwargs: python_result, + binding=messages_binding(lambda request: native_result), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=rules, + ) + + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.setattr(litellm, "ssl_security_level", None) + for setting in ("modify_params", "enable_anthropic_prompt_caching", "include_cost_in_streaming_usage"): + monkeypatch.setattr(litellm, setting, False) + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE", "ANTHROPIC_FEDERATION_RULE_ID", "ANTHROPIC_ORGANIZATION_ID"): + monkeypatch.delenv(name, raising=False) + monkeypatch.setenv("ANTHROPIC_API_KEY", "") + monkeypatch.delenv("ANTHROPIC_AUTH_TOKEN", raising=False) + assert served({}) is native_result + for name, value in settings.items(): + monkeypatch.setattr(litellm, name, value) + assert served(kwargs) is python_result + + +ASYNC_ONLY_FEATURES: Final = ( + pytest.param(Feature.PRE_REQUEST_HOOK, {}, {"callbacks": [_Rewrite()]}, id="pre-request-hook"), + pytest.param( + Feature.WEBSEARCH_INTERCEPTION, {}, {"callbacks": [WebSearchInterceptionLogger()]}, id="websearch-interception" + ), + pytest.param( + Feature.MESSAGES_INTERCEPTOR, + {"model": "deepseek/deepseek-v4-flash", "tools": [ADVISOR_TOOL]}, + {}, + id="messages-interceptor", + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("feature", "kwargs", "settings"), ASYNC_ONLY_FEATURES) +async def test_handler_hooks_the_sync_entry_never_runs_keep_only_async_calls_on_python( + monkeypatch: pytest.MonkeyPatch, feature: Feature, kwargs: Mapping[str, object], settings: Mapping[str, object] +) -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({feature})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + python_result: Final = response() + native_result: Final = response() + model: Final = str(kwargs.get("model", "anthropic/claude-sonnet-4-5")) + call_kwargs: Final = {name: value for name, value in kwargs.items() if name != "model"} + + async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + return python_result + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + monkeypatch.setattr(litellm, "callbacks", []) + for name, value in settings.items(): + monkeypatch.setattr(litellm, name, value) + served_async: Final = await _ADISPATCH.arun( + (16, MESSAGES, model), + call_kwargs, + python=python, + binding=amessages_binding(native), + native=lambda hook, request, args, native_kwargs: hook(request), + rules=rules, + ) + served_sync: Final = _DISPATCH.run( + (16, MESSAGES, model), + call_kwargs, + python=lambda *call_args, **python_kwargs: python_result, + binding=messages_binding(lambda request: native_result), + native=lambda hook, request, args, native_kwargs: hook(request), + rules=rules, + ) + assert served_async is python_result + assert served_sync is native_result + + +@pytest.mark.asyncio +async def test_interceptor_feature_follows_the_resolved_provider() -> None: + rules: Final[Rules] = ( + RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.MESSAGES_INTERCEPTOR})), + RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), + ) + native_result: Final = response() + + async def python(*call_args: object, **call_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub + pytest.fail("anthropic serves the advisor tool itself") + + async def native(request: NativeCall) -> AnthropicMessagesResponse: + return native_result + + result: Final = await _ADISPATCH.arun( + (16, MESSAGES, "claude-sonnet-4-5"), + {"tools": [ADVISOR_TOOL], "custom_llm_provider": "anthropic"}, + python=python, + binding=amessages_binding(native), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=rules, + ) + assert result is native_result diff --git a/tests/test_litellm_rust/responses/data.py b/tests/test_litellm_rust/responses/data.py index 604a7de7442..0347fd7a57b 100644 --- a/tests/test_litellm_rust/responses/data.py +++ b/tests/test_litellm_rust/responses/data.py @@ -69,5 +69,7 @@ async def call(server: RecordingServer, *, asynchronous: bool = True, **kwargs: SYNC_DEPLOYMENT_HOOK: Final = gap("native sync calls skip the async pre-call deployment hook") -STREAMING: Final = gap("native Responses streaming declines to Python", raises=RuntimeError) +STREAMING: Final = gap( + "native Responses streaming is unsupported, the catalog keeps it on Python", raises=litellm.APIConnectionError +) BODY_HOOKS: Final = gap("native Responses does not run the handler body hooks") diff --git a/tests/test_litellm_rust/responses/test_dispatch.py b/tests/test_litellm_rust/responses/test_dispatch.py index 69535a9e6f3..4498dbdf39d 100644 --- a/tests/test_litellm_rust/responses/test_dispatch.py +++ b/tests/test_litellm_rust/responses/test_dispatch.py @@ -5,20 +5,24 @@ from typing import Final, cast # noqa: TID251 # narrows legacy callable signat import pytest import litellm +from litellm.caching.caching import Cache +from litellm.integrations.custom_logger import CustomLogger from litellm.responses import dispatch as responses_dispatch from litellm.responses import main as python_responses from litellm.responses.dispatch import ( _ADISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch _DISPATCH, # pyright: ignore[reportPrivateUsage] # tests configured dispatch ) +from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.rust_bridge.bindings import NativeBinding -from litellm.rust_bridge.catalog import Route, RouteRule +from litellm.rust_bridge.catalog import Feature, Route, RouteRule from litellm.rust_bridge.configuration import Rollout from litellm.rust_bridge.public_call import NativeCall from litellm.rust_bridge.responses.entrypoints import ( NativeAresponses, NativeResponses, ) +from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_TOOL_TYPES from litellm.types.llms.openai import ResponsesAPIResponse INPUT: Final = [{"role": "user", "content": "hi"}] @@ -235,3 +239,167 @@ def test_positional_parameters_remain_available_to_native_projection() -> None: assert request.resolved["include"] is include assert request.resolved["instructions"] == "Be brief" assert request.resolved["max_output_tokens"] == 16 + + +class _Agentic(CustomLogger): + async def async_should_run_agentic_loop(self, *args: object, **kwargs: object) -> tuple[bool, dict]: # pyright: ignore[reportIncompatibleMethodOverride, reportMissingTypeArgument] # test double only needs the override + return False, {} + + +class _Deployment(CustomLogger): + async def async_pre_call_deployment_hook(self, *args: object, **kwargs: object) -> dict[str, object] | None: # pyright: ignore[reportIncompatibleMethodOverride] # test double only needs the override + return None + + +ENCODED_ID: Final = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-1", "rs_1") # pyright: ignore[reportPrivateUsage] # the encoder litellm stamps on earlier turns + + +@pytest.mark.parametrize( + ("feature", "kwargs", "settings"), + ( + (Feature.MOCK_RESPONSE, {"mock_response": "hi"}, {}), + (Feature.PROXY_MCP_TOOLS, {"tools": [{"type": "mcp", "server_label": "g", "server_url": "litellm_proxy"}]}, {}), + (Feature.CALLER_CLIENT, {"client": object()}, {}), + pytest.param(Feature.CALLER_CLIENT, {"shared_session": object()}, {}, id="shared-session"), + pytest.param(Feature.EXTRA_BODY, {"extra_body": {"stream": True}}, {}, id="extra-body-stream"), + pytest.param( + Feature.PROVIDER_AFFINITY, + {"provider_affinity_header": "X-Session-ID", "session_id": "session-1"}, + {}, + id="provider-affinity", + ), + pytest.param( + Feature.INPUT_REWRITE, + {"input": [{"type": "reasoning", "id": "rs_1", "summary": list[object](), "status": None}]}, + {}, + id="input-rewrite", + ), + (Feature.TIMEOUT_VARIANTS, {"request_timeout": 5}, {}), + (Feature.AGENTIC_LOOP_HOOK, {}, {"callbacks": [_Agentic()]}), + (Feature.TLS_POLICY, {}, {"ssl_security_level": "DEFAULT@SECLEVEL=1"}), + (Feature.DROP_PARAMS, {}, {"drop_params": True}), + (Feature.STREAMING, {"stream": True}, {}), + (Feature.PROMPT_MANAGEMENT, {"prompt_id": "welcome"}, {}), + (Feature.TEXT_FORMAT, {"text_format": dict}, {}), + (Feature.CHAT_COMPLETIONS_BRIDGE, {"use_chat_completions_api": True}, {}), + (Feature.CHAT_COMPLETIONS_BRIDGE, {"model": "openai/chat_completions/gpt-5"}, {}), + (Feature.RETRIES_FALLBACKS, {"num_retries": 2}, {}), + (Feature.DEPLOYMENT_RESPONSE_ID, {"litellm_metadata": {"model_info": {"id": "deployment-1"}}}, {}), + (Feature.ENCODED_ITEM_IDS, {"input": [{"type": "reasoning", "id": ENCODED_ID, "summary": []}]}, {}), + (Feature.MODIFY_PARAMS, {}, {"modify_params": True}), + (Feature.RESPONSE_CACHE, {}, {"cache": Cache()}), + ( + Feature.MANAGED_RESOURCE_IDS, + { + "input": [{"role": "user", "content": [{"type": "input_file", "file_id": "litellm-file"}]}], + "model_file_id_mapping": {"litellm-file": {"deployment-1": "file-abc"}}, + "model_info": {"id": "deployment-1"}, + }, + {}, + ), + (Feature.CACHE_CONTROL_MARKS, {"tools": [{"type": "function", "name": "f", "cache_control": {}}]}, {}), + ( + Feature.ANTHROPIC_TOOL_SEARCH, + {"tools": [{"type": sorted(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)[0], "name": "s"}]}, + {}, + ), + (Feature.TOOL_SCHEMA_REPAIR, {"tools": [{"type": "function", "name": "f", "parameters": "{}"}]}, {}), + (Feature.MODEL_PARAM_GATING, {"max_output_tokens": 1}, {}), + (Feature.CLIENT_HEADERS, {"headers": {"x-tenant": "a"}}, {}), + (Feature.DEPLOYMENT_HOOK, {}, {"callbacks": [_Deployment()]}), + ( + Feature.WORKLOAD_IDENTITY, + { + "openai_identity_provider_id": "idp-1", + "openai_service_account_id": "sa-1", + "openai_identity_token_file": "/var/run/token", + }, + {}, + ), + ), +) +def test_each_detected_feature_reaches_its_rule( + monkeypatch: pytest.MonkeyPatch, feature: Feature, kwargs: Mapping[str, object], settings: Mapping[str, object] +) -> None: + rules: Final = ( + RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY, features=frozenset({feature})), + RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED), + ) + python_result: Final = _response() + native_result: Final = _response() + + def served(call_kwargs: Mapping[str, object]) -> object: + return _DISPATCH.run( + (), + {"model": "gpt-5", "input": INPUT, **call_kwargs}, + python=lambda *call_args, **python_kwargs: python_result, + binding=responses_binding(lambda request: native_result), + native=lambda hook, request, call_args, native_kwargs: hook(request), + rules=rules, + ) + + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm, "modify_params", False) + monkeypatch.setattr(litellm, "ssl_security_level", None) + monkeypatch.setattr(litellm, "cache", None) + for setting in ("api_key", "openai_key", "api_base", "num_retries"): + monkeypatch.setattr(litellm, setting, None) + for name in ("SSL_SECURITY_LEVEL", "SSL_ECDH_CURVE", "OPENAI_API_KEY"): + monkeypatch.delenv(name, raising=False) + assert served({}) is native_result + for name, value in settings.items(): + monkeypatch.setattr(litellm, name, value) + assert served(kwargs) is python_result + + +@pytest.mark.asyncio +async def test_deployment_hook_keeps_only_sync_calls_on_python(monkeypatch: pytest.MonkeyPatch) -> None: + rules: Final = ( + RouteRule(Route.RESPONSES, Rollout.PYTHON_ONLY, features=frozenset({Feature.DEPLOYMENT_HOOK})), + RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED), + ) + native_result: Final = _response() + + async def python(*call_args: object, **call_kwargs: object) -> ResponsesAPIResponse: # kwargs-ok: stub + pytest.fail("the async handler leaves the deployment hook to the shared call wrapper") + + async def native(request: NativeCall) -> ResponsesAPIResponse: + return native_result + + monkeypatch.setattr(litellm, "callbacks", [_Deployment()]) + result: Final = await _ADISPATCH.arun( + (), + {"model": "gpt-5", "input": INPUT}, + python=python, + binding=aresponses_binding(native), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=rules, + ) + assert result is native_result + + +@pytest.mark.parametrize( + ("kwargs", "native"), + ( + ({"model": "gpt-5"}, True), + ({"model": "openai/gpt-5"}, True), + ({"model": "gpt-5", "custom_llm_provider": "openai"}, True), + ({"model": "anthropic/claude-sonnet-4-5"}, False), + ({"model": "claude-sonnet-4-5"}, False), + ({"model": "gpt-5", "custom_llm_provider": "azure"}, False), + ), +) +def test_provider_rule_sees_the_resolved_provider(kwargs: Mapping[str, object], native: bool) -> None: + python_result: Final = _response() + native_result: Final = _response() + + result: Final = _DISPATCH.run( + (), + {"input": INPUT, **kwargs}, + python=lambda *call_args, **call_kwargs: python_result, + binding=responses_binding(lambda request: native_result), + native=lambda hook, request, call_args, call_kwargs: hook(request), + rules=(RouteRule(Route.RESPONSES, Rollout.RUST_REQUIRED, providers=frozenset({"openai"})),), + ) + assert result is (native_result if native else python_result) diff --git a/tests/test_litellm_rust/responses/test_failures.py b/tests/test_litellm_rust/responses/test_failures.py index 6dad1bc7278..4c4bb96c6ac 100644 --- a/tests/test_litellm_rust/responses/test_failures.py +++ b/tests/test_litellm_rust/responses/test_failures.py @@ -56,28 +56,3 @@ async def test_provider_failure_is_terminal_and_shared_with_callbacks( assert len(failure) == 1 assert _OBJECT.validate_python(failure[0].kwargs)["exception"] is caught.value assert not any("success" in name for name in recorder.names) - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "options", - ( - {"stream": True}, - {"extra_body": {"provider_option": True}}, - {"mock_response": "mock"}, - {"num_retries": 1}, - {"use_chat_completions_api": True}, - {"model_list": []}, - ), -) -async def test_unsupported_requests_fail_before_callbacks_and_provider_io( - server: RecordingServer, options: dict[str, object] -) -> None: - recorder: Final = RecordingLogger() - server.expected_requests = 0 - - with pytest.raises(RuntimeError, match="declined the request"): - await call(server, callbacks=[recorder], **options) - - assert not server.requests - assert not recorder.events diff --git a/tests/test_litellm_rust/test_dispatch.py b/tests/test_litellm_rust/test_dispatch.py index 94526e83fbb..193866409a2 100644 --- a/tests/test_litellm_rust/test_dispatch.py +++ b/tests/test_litellm_rust/test_dispatch.py @@ -1,6 +1,6 @@ from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping from dataclasses import dataclass -from typing import Final, TypeAlias +from typing import Final, NoReturn, TypeAlias import pytest @@ -245,6 +245,50 @@ async def dispatch_without_python( return await dispatch.arun(("model",), {"page": 1}, python=NO_PYTHON, binding=bound, native=native, rules=rules) +def failing_context(request: Request) -> RouteContext: + raise RuntimeError("detector exploded") + + +def failing_dispatch() -> PublicDispatch[Request]: + return PublicDispatch( + route=Route.CHAT_COMPLETIONS, request=lambda args, kwargs: Request(model="model"), context=failing_context + ) + + +def reject_native(hook: object, request: Request, args: tuple[object, ...], kwargs: Mapping[str, object]) -> NoReturn: + pytest.fail("a failed context build must not reach native") + + +def test_a_detector_error_keeps_the_call_on_python() -> None: + expected: Final = object() + result: Final = failing_dispatch().run( + ("model",), + {}, + python=lambda *args, **kwargs: expected, + binding=binding(), + native=reject_native, + rules=(RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),), + ) + assert result is expected + + +async def test_a_detector_error_keeps_the_async_call_on_python() -> None: + expected: Final = object() + + async def python(*args: object, **kwargs: object) -> object: # kwargs-ok: stub + return expected + + result: Final = await failing_dispatch().arun( + ("model",), + {}, + python=python, + binding=binding(), + native=reject_native, + rules=(RouteRule(Route.CHAT_COMPLETIONS, Rollout.RUST_REQUIRED),), + ) + assert result is expected + + def ocr_dispatch(request: Request | None) -> PublicDispatch[Request]: return PublicDispatch( route=Route.OCR, diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py new file mode 100644 index 00000000000..726128386d5 --- /dev/null +++ b/tests/unit/rust_bridge/test_detectors.py @@ -0,0 +1,522 @@ +from collections.abc import Callable, Mapping +from copy import deepcopy +from datetime import datetime +from functools import reduce +from typing import Final + +import httpx +import pytest + +import litellm +from litellm.caching.caching import Cache +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.rust_bridge import detectors +from litellm.rust_bridge.chat_completions import detectors as chat_detectors +from litellm.rust_bridge.messages import detectors as messages_detectors +from litellm.rust_bridge.public_call import NativeCall +from litellm.rust_bridge.responses import detectors as responses_detectors +from litellm.types.llms.anthropic import ANTHROPIC_ADVISOR_TOOL_TYPE, ANTHROPIC_TOOL_SEARCH_TOOL_TYPES + +GATEWAY_TOOL: Final = {"type": "mcp", "server_label": "gateway", "server_url": "litellm_proxy"} +REMOTE_TOOL: Final = {"type": "mcp", "server_label": "docs", "server_url": "https://mcp.example.com/sse"} +ENCODED_ID: Final = ResponsesAPIRequestUtils._build_encrypted_item_id("deployment-1", "rs_1") # pyright: ignore[reportPrivateUsage] # the encoder litellm stamps on earlier turns +WRAPPED_CONTENT: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id("opaque", "deployment-1") + + +MAPPED_FILE: Final = { + "model_file_id_mapping": {"litellm-file": {"deployment-1": "file-abc"}}, + "model_info": {"id": "deployment-1"}, +} +MAPPED_FILE_INPUT: Final = [{"role": "user", "content": [{"type": "input_file", "file_id": "litellm-file"}]}] +CACHED_TEXT: Final = {"type": "input_text", "text": "hi", "cache_control": {"type": "ephemeral"}} +THINKING_TURN: Final = {"role": "assistant", "content": [{"type": "thinking", "thinking": "x", "signature": "s"}]} +REDACTED: Final = {"type": "redacted_thinking", "data": "x"} +TOOL_SEARCH_TYPE: Final = sorted(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)[0] + + +def function_tool(parameters: object) -> Mapping[str, object]: + return {"type": "function", "name": "f", "parameters": parameters} + + +def call(**kwargs: object) -> NativeCall: + return NativeCall(args=(), kwargs=kwargs, base={"model": "gpt-5"}) + + +@pytest.mark.parametrize( + ("model", "provider", "tools", "expected"), + ( + ("deepseek/test-model", None, [{"type": ANTHROPIC_ADVISOR_TOOL_TYPE}], True), + ("anthropic/test-model", None, [{"type": ANTHROPIC_ADVISOR_TOOL_TYPE}], False), + ("test-model", "anthropic", [{"type": ANTHROPIC_ADVISOR_TOOL_TYPE}], False), + ("unresolved-model", None, [{"type": ANTHROPIC_ADVISOR_TOOL_TYPE}], True), + ("deepseek/test-model", None, [{"type": "function", "name": "f"}], False), + ("deepseek/test-model", None, ["not-a-tool"], False), + ("deepseek/test-model", None, [], False), + ("deepseek/test-model", None, None, False), + ), +) +def test_messages_interception_matches_tools_and_resolved_provider( + model: str, provider: str | None, tools: object, expected: bool +) -> None: + assert messages_detectors.intercepted(call(model=model, custom_llm_provider=provider, tools=tools)) is expected + + +def _wrap_nested_level(value: object, depth: int) -> object: + return {"nested": value} if depth % 2 else [value] + + +@pytest.mark.parametrize( + ("detector", "kwargs", "expected"), + ( + (detectors.mock_response, {"mock_response": "hi"}, True), + (detectors.mock_response, {"mock_response": None}, False), + (detectors.mock_response, {"mock_tool_calls": [{"id": "call_1"}]}, False), + (chat_detectors.chat_mock_response, {"mock_response": "hi"}, True), + (chat_detectors.chat_mock_response, {"mock_tool_calls": [{"id": "call_1"}]}, True), + (chat_detectors.chat_mock_response, {"mock_timeout": True}, True), + (chat_detectors.chat_mock_response, {"mock_tool_calls": [], "mock_timeout": False}, False), + (responses_detectors.client_headers, {"headers": {"x-tenant": "a"}}, True), + (responses_detectors.client_headers, {"headers": {}}, False), + (responses_detectors.client_headers, {"headers": None}, False), + ( + chat_detectors.model_list, + {"model_list": [{"model_name": "gpt-5", "litellm_params": {"model": "gpt-5"}}]}, + True, + ), + (chat_detectors.model_list, {"model_list": None}, False), + (chat_detectors.chat_retries_fallbacks, {"context_window_fallback_dict": {"gpt-5": "gpt-5-mini"}}, True), + (chat_detectors.chat_retries_fallbacks, {"fallbacks": ["gpt-5-mini"]}, True), + (detectors.retries_fallbacks, {"context_window_fallback_dict": {"gpt-5": "gpt-5-mini"}}, False), + (detectors.prompt_management, {"model": "litellm_agent/gpt-5"}, True), + (detectors.prompt_management, {"model": "gpt-5"}, False), + (detectors.caller_client, {"client": object()}, True), + (detectors.caller_client, {"shared_session": object()}, True), + (detectors.caller_client, {"client": None, "shared_session": None}, False), + (detectors.caller_client, {}, False), + (detectors.extra_body, {"extra_body": {"stream": True}}, True), + (detectors.extra_body, {"extra_body": {"stream": False}}, True), + (detectors.extra_body, {"extra_body": {}}, False), + (detectors.extra_body, {"extra_body": None}, False), + (detectors.extra_body, {}, False), + ( + responses_detectors.provider_affinity, + {"provider_affinity_header": "X-Session-ID", "session_id": "session-1"}, + True, + ), + (responses_detectors.provider_affinity, {"provider_affinity_header": "X-Session-ID"}, True), + ( + responses_detectors.provider_affinity, + {"provider_affinity_header": "X-Session-ID", "extra_headers": {"x-session-id": "session-1"}}, + True, + ), + (responses_detectors.provider_affinity, {"provider_affinity_header": "Authorization"}, True), + (responses_detectors.provider_affinity, {"provider_affinity_header": "invalid header"}, True), + (responses_detectors.provider_affinity, {"provider_affinity_header": ""}, True), + (responses_detectors.provider_affinity, {"provider_affinity_header": None, "session_id": "session-1"}, False), + (responses_detectors.provider_affinity, {}, False), + (detectors.streaming, {"stream": True}, True), + (detectors.streaming, {"stream": False}, False), + (detectors.timeout_variants, {"request_timeout": 5}, True), + (detectors.timeout_variants, {"stream": True, "stream_timeout": 5}, True), + (detectors.timeout_variants, {"stream_timeout": 5}, False), + (detectors.timeout_variants, {"timeout": httpx.Timeout(60, connect=2)}, True), + (detectors.timeout_variants, {"timeout": 60}, False), + (detectors.proxy_mcp_tools, {"tools": [GATEWAY_TOOL]}, True), + (detectors.proxy_mcp_tools, {"tools": [REMOTE_TOOL]}, False), + (detectors.proxy_mcp_tools, {"tools": None}, False), + (detectors.prompt_management, {"prompt_id": "welcome"}, True), + (detectors.prompt_management, {"prompt_variables": {"name": "x"}}, False), + (responses_detectors.text_format, {"text_format": dict}, True), + (responses_detectors.text_format, {}, False), + (responses_detectors.chat_completions_bridge, {"use_chat_completions_api": True}, True), + (responses_detectors.chat_completions_bridge, {"use_chat_completions_api": False}, False), + (responses_detectors.chat_completions_bridge, {"model": "openai/chat_completions/gpt-5"}, True), + (responses_detectors.chat_completions_bridge, {"model": "openai/gpt-5"}, False), + (detectors.retries_fallbacks, {"num_retries": 0}, True), + (detectors.retries_fallbacks, {"max_retries": 2}, True), + (detectors.retries_fallbacks, {"fallbacks": ["gpt-5-mini"]}, True), + (detectors.retries_fallbacks, {"retry_policy": {"TimeoutErrorRetries": 2}}, True), + (detectors.retries_fallbacks, {}, False), + (detectors.prompt_management, {"knowledge_bases": ["kb-1"]}, True), + (detectors.prompt_management, {"vector_store_ids": []}, False), + (responses_detectors.managed_resource_ids, {"input": MAPPED_FILE_INPUT, **MAPPED_FILE}, True), + (responses_detectors.managed_resource_ids, {"input": MAPPED_FILE_INPUT}, False), + (responses_detectors.managed_resource_ids, {"input": "hi"}, False), + (responses_detectors.cache_control_marks, {"input": [{"role": "user", "content": [CACHED_TEXT]}]}, True), + ( + responses_detectors.cache_control_marks, + {"tools": [{"type": "function", "name": "f", "cache_control": {}}]}, + True, + ), + ( + responses_detectors.cache_control_marks, + {"input": [{"role": "user", "content": [{"type": "input_text"}]}]}, + False, + ), + ( + responses_detectors.anthropic_tool_search, + {"tools": [{"type": TOOL_SEARCH_TYPE, "name": "tool_search"}]}, + True, + ), + (responses_detectors.anthropic_tool_search, {"tools": [{"type": "function", "name": "f"}]}, False), + (responses_detectors.tool_schema_repair, {"tools": [function_tool('{"type": "object"}')]}, True), + ( + responses_detectors.tool_schema_repair, + {"tools": [function_tool({"type": "string", "pattern": r"\p{L}"})]}, + True, + ), + (responses_detectors.tool_schema_repair, {"tools": [function_tool({"type": "object"})]}, False), + (responses_detectors.tool_schema_repair, {}, False), + (responses_detectors.model_param_gating, {"max_output_tokens": 1}, True), + (responses_detectors.model_param_gating, {"max_output_tokens": 1024}, False), + (detectors.cache_control_injection, {"enable_prompt_caching": True}, True), + (detectors.cache_control_injection, {"cache_control_injection_points": [{"location": "message"}]}, True), + (detectors.cache_control_injection, {"cache_control_injection_points": []}, False), + (messages_detectors.thinking_signature_retry, {"messages": [THINKING_TURN]}, True), + ( + messages_detectors.thinking_signature_retry, + {"messages": [{"role": "assistant", "content": [REDACTED]}]}, + True, + ), + (messages_detectors.thinking_signature_retry, {"messages": [{"role": "assistant", "content": "done"}]}, False), + ( + messages_detectors.thinking_signature_retry, + {"messages": [{"role": "user", "content": [{"type": "text"}]}]}, + False, + ), + (detectors.drop_params, {"drop_params": True}, True), + (detectors.drop_params, {"drop_params": False}, False), + (detectors.drop_params, {"drop_params": " True "}, True), + (detectors.drop_params, {"drop_params": "false"}, False), + (detectors.drop_params, {"drop_params": "maybe"}, False), + (detectors.drop_params, {"additional_drop_params": ["metadata.user_id"]}, True), + (detectors.drop_params, {"additional_drop_params": []}, False), + ( + responses_detectors.deployment_response_id, + {"litellm_metadata": {"model_info": {"id": "deployment-1"}}}, + True, + ), + (responses_detectors.deployment_response_id, {"litellm_metadata": {"model_info": {}}}, False), + (responses_detectors.deployment_response_id, {"litellm_metadata": None}, False), + (responses_detectors.encoded_item_ids, {"input": [{"type": "reasoning", "id": ENCODED_ID}]}, True), + ( + responses_detectors.encoded_item_ids, + {"input": [{"type": "reasoning", "encrypted_content": WRAPPED_CONTENT}]}, + True, + ), + ( + responses_detectors.encoded_item_ids, + {"input": [{"type": "reasoning", "id": "rs_1", "encrypted_content": "x"}]}, + False, + ), + (responses_detectors.encoded_item_ids, {"input": "hi"}, False), + ), +) +def test_request_detectors( + detector: Callable[[NativeCall], bool], kwargs: Mapping[str, object], expected: bool +) -> None: + assert detector(call(**kwargs)) is expected + + +def test_cache_control_marks_handles_deeply_nested_input() -> None: + nested_input: Final = reduce(_wrap_nested_level, range(5_000), {"cache_control": {}}) + assert responses_detectors.cache_control_marks(call(input=nested_input)) + + +@pytest.mark.parametrize( + ("input_value", "expected"), + ( + pytest.param( + [{"type": "reasoning", "id": "rs_00000000-0000-4000-8000-000000000000", "summary": list[object]()}], + True, + id="bridge-minted-reasoning", + ), + pytest.param( + [{"type": "reasoning", "encrypted_content": '[{"type":"thinking","thinking":"x","signature":"s"}]'}], + True, + id="bridge-thinking-blocks", + ), + pytest.param( + [{"type": "function_call", "id": "foreign-id", "call_id": "call_1", "name": "f", "arguments": "{}"}], + True, + id="foreign-tool-call-id", + ), + pytest.param( + [{"type": "reasoning", "id": "rs_1", "summary": list[object](), "status": None}], + True, + id="null-reasoning-status", + ), + pytest.param( + [{"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "f", "arguments": "{}"}], + False, + id="native-tool-call-id", + ), + pytest.param( + [{"type": "custom_tool_call", "call_id": "call_1", "name": "f", "input": "x", "namespace": "ns"}], + True, + id="custom-tool-call-namespace", + ), + pytest.param( + [{"type": "custom_tool_call", "call_id": "call_1", "name": "f", "input": "x"}], + False, + id="custom-tool-call", + ), + pytest.param([{"type": "reasoning", "id": "rs_1", "summary": list[object]()}], False, id="native-reasoning"), + pytest.param([{"role": "user", "content": "hi"}], False, id="message"), + pytest.param("hi", False, id="string"), + pytest.param([], False, id="empty"), + pytest.param(None, False, id="absent"), + ), +) +def test_input_rewrite_preserves_caller_input(input_value: object, expected: bool) -> None: + original: Final = deepcopy(input_value) + assert responses_detectors.input_rewrite(call(input=input_value)) is expected + assert input_value == original + + +@pytest.mark.parametrize( + ("detector", "setting", "value", "kwargs"), + ( + (detectors.drop_params, "drop_params", True, {}), + (detectors.modify_params, "modify_params", True, {}), + (detectors.retries_fallbacks, "num_retries", 3, {}), + (detectors.cache_control_injection, "enable_anthropic_prompt_caching", True, {}), + (messages_detectors.stream_cost_injection, "include_cost_in_streaming_usage", True, {"stream": True}), + (messages_detectors.stream_cost_injection, "include_cost_in_streaming_usage", "true", {"stream": True}), + (chat_detectors.chat_retries_fallbacks, "model_fallbacks", [{"gpt-5": ["gpt-5-mini"]}], {}), + (chat_detectors.chat_retries_fallbacks, "num_retries", 3, {}), + (chat_detectors.model_alias_map, "model_alias_map", {"gpt-5": "openai/gpt-5"}, {}), + ), +) +def test_global_settings_are_detected( + monkeypatch: pytest.MonkeyPatch, + detector: Callable[[NativeCall], bool], + setting: str, + value: object, + kwargs: Mapping[str, object], +) -> None: + for name in ("drop_params", "modify_params", "enable_anthropic_prompt_caching", "include_cost_in_streaming_usage"): + monkeypatch.setattr(litellm, name, False) + for name in ("num_retries", "model_fallbacks"): + monkeypatch.setattr(litellm, name, None) + monkeypatch.setattr(litellm, "model_alias_map", {}) + assert not detector(call(**kwargs)) + monkeypatch.setattr(litellm, setting, value) + assert detector(call(**kwargs)) + + +def test_stream_cost_injection_needs_a_stream(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "include_cost_in_streaming_usage", True) + assert not messages_detectors.stream_cost_injection(call()) + + +def test_model_alias_map_applies_only_to_the_requested_model(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "model_alias_map", {"fast": "openai/gpt-5-mini"}) + assert chat_detectors.model_alias_map(call(model="fast")) + assert not chat_detectors.model_alias_map(call()) + + +@pytest.mark.parametrize( + ("kwargs", "expected"), + ( + pytest.param({"temperature": 0.2}, True, id="gpt-5-temperature"), + pytest.param({"max_tokens": 16}, False, id="gpt-5-max-tokens"), + pytest.param({"model": "gpt-4o", "temperature": 0.2}, False, id="gpt-4o-temperature"), + pytest.param({"model": "anthropic/claude-sonnet-4-5", "logit_bias": {"1": 1}}, True, id="unsupported-param"), + pytest.param({"model": "anthropic/claude-sonnet-4-5", "n": 1}, False, id="langchain-default-n"), + pytest.param({"model": "anthropic/claude-sonnet-4-5", "n": 2}, True, id="unsupported-n"), + pytest.param({"model": "anthropic/claude-sonnet-4-5", "max_tokens": 16, "stream": True}, False, id="supported"), + pytest.param({"model": "unknown-model", "temperature": 0.2}, False, id="unresolved-provider"), + pytest.param({}, False, id="no-params"), + ), +) +def test_chat_model_param_gating_follows_the_provider_param_rules( + monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object], expected: bool +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + assert chat_detectors.chat_model_param_gating(call(**kwargs)) is expected + + +def test_openai_workload_identity_applies_only_without_a_static_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.setattr(litellm, "openai_key", None) + monkeypatch.setattr(litellm, "api_base", None) + identity: Final = { + "openai_identity_provider_id": "idp-1", + "openai_service_account_id": "sa-1", + "openai_identity_token_file": "/var/run/token", + } + assert responses_detectors.openai_workload_identity(call(**identity)) + assert not responses_detectors.openai_workload_identity(call(**identity, api_key="sk-static")) + assert not responses_detectors.openai_workload_identity(call()) + + +def test_anthropic_workload_identity_applies_only_without_a_static_key(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "ANTHROPIC_FEDERATION_RULE_ID", + "ANTHROPIC_ORGANIZATION_ID", + ): + monkeypatch.delenv(name, raising=False) + federation: Final = {"anthropic_federation_rule_id": "rule-1", "anthropic_organization_id": "org-1"} + assert messages_detectors.anthropic_workload_identity(call(**federation)) + assert not messages_detectors.anthropic_workload_identity(call(**federation, api_key="sk-ant-static")) + assert not messages_detectors.anthropic_workload_identity(call(anthropic_federation_rule_id="rule-1")) + + +class _Logger(CustomLogger): + pass + + +class _Rewrite(CustomLogger): + async def async_pre_request_hook(self, model: str, messages: list, kwargs: dict) -> dict | None: # pyright: ignore[reportMissingTypeArgument] # mirrors the CustomLogger signature + return kwargs + + +class _InheritedRewrite(_Rewrite): + pass + + +class _Agentic(CustomLogger): + async def async_should_run_agentic_loop(self, *args: object, **kwargs: object) -> tuple[bool, dict]: # pyright: ignore[reportIncompatibleMethodOverride, reportMissingTypeArgument] # test double only needs the override + return False, {} + + +class _ChatAgentic(CustomLogger): + async def async_should_run_chat_completion_agentic_loop( + self, *args: object, **kwargs: object + ) -> tuple[bool, dict[str, object]]: # pyright: ignore[reportIncompatibleMethodOverride] # test double only needs the override + return False, {} + + +class _Deployment(CustomLogger): + async def async_pre_call_deployment_hook(self, *args: object, **kwargs: object) -> dict[str, object] | None: # pyright: ignore[reportIncompatibleMethodOverride] # test double only needs the override + return None + + +@pytest.mark.parametrize( + ("detector", "callbacks", "expected"), + ( + (messages_detectors.pre_request_hook, [_Rewrite()], True), + (messages_detectors.pre_request_hook, [_InheritedRewrite()], True), + (messages_detectors.pre_request_hook, [_Logger(), _Agentic()], False), + (messages_detectors.pre_request_hook, ["not-a-logger"], False), + (detectors.agentic_loop_hook, [_Agentic()], True), + (detectors.agentic_loop_hook, [_Logger(), _Rewrite()], False), + (detectors.agentic_loop_hook, [_ChatAgentic()], False), + (chat_detectors.chat_agentic_loop_hook, [_ChatAgentic()], True), + (chat_detectors.chat_agentic_loop_hook, [_Agentic()], True), + (chat_detectors.chat_agentic_loop_hook, [_Logger(), _Deployment()], False), + (responses_detectors.deployment_hook, [_Deployment()], True), + (responses_detectors.deployment_hook, [_Logger(), _Agentic(), _Rewrite()], False), + ), +) +def test_callback_detectors_need_an_override( + monkeypatch: pytest.MonkeyPatch, detector: Callable[[NativeCall], bool], callbacks: list[object], expected: bool +) -> None: + monkeypatch.setattr(litellm, "callbacks", callbacks) + assert detector(call()) is expected + + +def _logging_with(callback: CustomLogger) -> Logging: + logging_obj: Final = Logging( + model="claude-sonnet-4-5", + messages=[], + stream=False, + call_type="anthropic_messages", + start_time=datetime.now(), + litellm_call_id="call-1", + function_id="function-1", + ) + logging_obj.add_dynamic_callback(callback) + return logging_obj + + +@pytest.mark.parametrize( + ("detector", "kwargs", "expected"), + ( + (detectors.agentic_loop_hook, {"success_callback": [_Agentic()]}, True), + (detectors.agentic_loop_hook, {"litellm_logging_obj": _logging_with(_Agentic())}, True), + (detectors.agentic_loop_hook, {"success_callback": [_Logger()]}, False), + (messages_detectors.pre_request_hook, {"success_callback": [_Rewrite()]}, False), + (chat_detectors.chat_agentic_loop_hook, {"litellm_logging_obj": _logging_with(_ChatAgentic())}, True), + (responses_detectors.deployment_hook, {"success_callback": [_Deployment()]}, True), + ), + ids=( + "success-callback", + "logging-object", + "plain-logger", + "pre-request-reads-globals-only", + "chat-agentic-logging-object", + "deployment-success-callback", + ), +) +def test_callback_detectors_read_the_request_callbacks_the_handler_reads( + monkeypatch: pytest.MonkeyPatch, + detector: Callable[[NativeCall], bool], + kwargs: Mapping[str, object], + expected: bool, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + assert detector(call(**kwargs)) is expected + + +def test_websearch_interception_needs_the_interception_logger(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.integrations.websearch_interception.handler import WebSearchInterceptionLogger + + monkeypatch.setattr(litellm, "callbacks", [_Rewrite()]) + assert not messages_detectors.websearch_interception(call()) + monkeypatch.setattr(litellm, "callbacks", [WebSearchInterceptionLogger()]) + assert messages_detectors.websearch_interception(call()) + + +@pytest.mark.parametrize( + ("cache", "kwargs", "expected"), + ( + (None, {}, False), + (Cache(), {}, True), + (Cache(), {"cache": {"ttl": 60}}, True), + (Cache(), {"cache": {"no-cache": True}}, False), + ), +) +def test_response_cache_is_skipped_only_by_no_cache( + monkeypatch: pytest.MonkeyPatch, cache: Cache | None, kwargs: Mapping[str, object], expected: bool +) -> None: + monkeypatch.setattr(litellm, "cache", cache) + assert detectors.response_cache(call(**kwargs)) is expected + + +@pytest.mark.parametrize( + ("environment", "setting", "expected"), + ((None, None, False), ("DEFAULT@SECLEVEL=1", None, True), (None, "DEFAULT@SECLEVEL=1", True)), +) +def test_tls_policy_reads_the_security_level_environment_then_the_setting( + monkeypatch: pytest.MonkeyPatch, environment: str | None, setting: str | None, expected: bool +) -> None: + monkeypatch.delenv("SSL_SECURITY_LEVEL", raising=False) + monkeypatch.delenv("SSL_ECDH_CURVE", raising=False) + monkeypatch.setattr(litellm, "ssl_ecdh_curve", None) + if environment is not None: + monkeypatch.setenv("SSL_SECURITY_LEVEL", environment) + monkeypatch.setattr(litellm, "ssl_security_level", setting) + assert detectors.tls_policy(call()) is expected + + +@pytest.mark.parametrize( + ("environment", "setting", "expected"), + ((None, "X25519", False), (None, " P-256 ", False), ("secp521r1", None, True), (None, "secp521r1", True)), +) +def test_tls_policy_flags_only_curves_native_tls_lacks( + monkeypatch: pytest.MonkeyPatch, environment: str | None, setting: str | None, expected: bool +) -> None: + monkeypatch.delenv("SSL_SECURITY_LEVEL", raising=False) + monkeypatch.delenv("SSL_ECDH_CURVE", raising=False) + monkeypatch.setattr(litellm, "ssl_security_level", None) + if environment is not None: + monkeypatch.setenv("SSL_ECDH_CURVE", environment) + monkeypatch.setattr(litellm, "ssl_ecdh_curve", setting) + assert detectors.tls_policy(call()) is expected From 2cba80cd0deaec8b4151da72b135e101ac06f5c0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 15:11:26 -0700 Subject: [PATCH 007/235] chore(cost-map): add search_free_requests to the model prices schema (#47635) Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- model_prices_and_context_window.schema.json | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 0e8f4f596c0..016d40ea874 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -954,6 +954,33 @@ }, "additionalProperties": false }, + "search_free_requests": { + "type": "object", + "description": "Free Google Search grounding requests per period, shared across models in the same shared_group.", + "required": [ + "count", + "period", + "shared_group" + ], + "additionalProperties": false, + "properties": { + "count": { + "type": "integer", + "minimum": 0 + }, + "period": { + "type": "string", + "enum": [ + "day", + "month" + ] + }, + "shared_group": { + "type": "string", + "pattern": "\\S" + } + } + }, "source": { "type": "string", "description": "URL of the provider pricing/model page this entry was taken from." From ffb7133d326e979506f17abee0412ca66de53d3a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 15:11:45 -0700 Subject: [PATCH 008/235] fix(proxy): price a 499 spend row only for the proxy's own disconnect, and never after the call already completed (#47652) * fix(proxy): price a 499 spend row only for the proxy's own disconnect, and never after the call already completed * fix(proxy): gate the zero-cost 499 row on the success log being enqueued, not merely claimed --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 11 ++ litellm/proxy/utils.py | 11 +- litellm/utils.py | 1 + ..._utils_model_creation_and_error_logging.py | 119 ++++++++++++++++++ 4 files changed, 138 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 37df750b555..069dcffd21a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -796,6 +796,7 @@ class Logging(LiteLLMLoggingBaseClass): self.defer_async_logging: bool = False self.enqueue_deferred_logging: Callable[[], None] | None = None self._async_success_scheduled: bool = False + self._async_success_enqueued: bool = False self._sync_success_scheduled: bool = False self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None @@ -2480,6 +2481,16 @@ class Logging(LiteLLMLoggingBaseClass): except Exception: return True + @property + def async_success_log_enqueued(self) -> bool: + """True once the success log task was actually created, not merely claimed, so a later failure raised + around it (a disconnect cancelling a still-pending guardrail task) must not bill the prompt a second + time. A claimed log parked in the deferred slot never runs once a 499 interrupts, so it stays False.""" + return self._async_success_enqueued + + def mark_async_success_log_enqueued(self) -> None: + self._async_success_enqueued = True + def claim_async_success_log(self) -> bool: """One async success log per request. The innermost @client wrapper always exits first, since the outer one is awaiting it, so it claims the log here and the outer wrapper gets diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 147f9292a21..aca222c31ef 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1386,8 +1386,9 @@ def _timed_out_reading_from_provider(exception: Exception) -> bool: def _client_disconnected(exception: Exception) -> bool: - """The 499 the proxy raises when the caller hung up while the provider was still working on the request.""" - return getattr(exception, "status_code", None) == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + """The 499 the proxy itself raises when the caller hung up while the provider was still working on the + request. A provider's own 499 arrives as a litellm exception and is a plain upstream failure.""" + return isinstance(exception, HTTPException) and exception.status_code == LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED def _provider_billed_the_failure(exception: Exception) -> bool: @@ -1409,17 +1410,19 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: _litellm_params: Final = getattr(_logging_obj, "litellm_params", None) _base_model: Final = get_base_model_from_metadata(model_call_details=_model_call_details) _deployment_id: Final = _logging_obj.get_router_model_id() if isinstance(_logging_obj, Logging) else None + _success_already_logged: Final = isinstance(_logging_obj, Logging) and _logging_obj.async_success_log_enqueued + _billed: Final = provider_billed and not _success_already_logged _usage_to_lift: Final = _failure_usage_to_lift( model_call_details=_model_call_details, request_body=request_data, dispatched=_first_handoff is not None, - provider_billed=provider_billed, + provider_billed=_billed, deployment_id=_deployment_id, base_model=_base_model, ) _prompt_estimate: Final = ( _estimated_prompt_usage(_model_call_details, request_data) - if provider_billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage") + if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage") else None ) _earlier_attempts: Final = ( diff --git a/litellm/utils.py b/litellm/utils.py index 6b16fc398bd..987139b7530 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1500,6 +1500,7 @@ def _schedule_async_success_logging( return def _enqueue_async_logging() -> None: + logging_obj.mark_async_success_log_enqueued() asyncio.create_task( _client_async_logging_helper( logging_obj=logging_obj, diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 48ad3490ebb..960bbc9cba3 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -828,6 +828,125 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: assert expected_input_cost > 0 assert request_data["response_cost"] == pytest.approx(expected_input_cost) + @pytest.mark.asyncio + async def test_upstream_499_error_is_a_plain_failure_not_a_priced_disconnect(self): + """A provider answering HTTP 499 surfaces as litellm.APIError with status_code 499. Only the + proxy's own disconnect HTTPException means the caller hung up mid-generation, so the upstream + error keeps the estimated usage at zero cost like any other post-dispatch provider error.""" + from datetime import datetime + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + from litellm.types.utils import Usage + + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 0), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ), + "metadata": {}, + } + await self._run( + request_data, + original_exception=litellm.APIError( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, + message="client closed request", + llm_provider="openai", + model="gpt-3.5-turbo", + ), + ) + + estimated = request_data["combined_usage_object"] + assert isinstance(estimated, Usage) + assert estimated.prompt_tokens > 0 + assert request_data["response_cost"] == 0.0 + + @pytest.mark.asyncio + async def test_client_disconnect_after_the_call_already_completed_is_not_priced_again(self): + """The LLM task finished and claimed its success log (which carries the real usage and cost) + while a during_call guardrail task was still pending, then the caller hung up. The 499 raised + for that disconnect must not price the prompt a second time.""" + from datetime import datetime + + from fastapi import HTTPException + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "count these input tokens please"}], + stream=False, + call_type="acompletion", + start_time=datetime(2026, 1, 1, 12, 0, 0), + litellm_call_id="completed-before-disconnect", + function_id="fn", + ) + logging_obj.model_call_details.update( + { + "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ) + assert logging_obj.claim_async_success_log() + logging_obj.mark_async_success_log_enqueued() + request_data = {"litellm_logging_obj": logging_obj, "metadata": {}} + await self._run( + request_data, + original_exception=HTTPException( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request" + ), + ) + + assert request_data.get("response_cost", 0.0) == 0.0 + + @pytest.mark.asyncio + async def test_client_disconnect_with_a_claimed_but_deferred_success_log_is_still_priced(self): + """With a post_call guardrail registered the success log is claimed but parked in the deferred slot, + and a 499 raised before the flush means it never runs. Nothing else bills that request, so the + failure row keeps the estimated input cost.""" + from datetime import datetime + + from fastapi import HTTPException + + from litellm.constants import LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED + from litellm.litellm_core_utils.litellm_logging import Logging + + logging_obj = Logging( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "count these input tokens please"}], + stream=False, + call_type="acompletion", + start_time=datetime(2026, 1, 1, 12, 0, 0), + litellm_call_id="deferred-success-never-flushed", + function_id="fn", + ) + logging_obj.model_call_details.update( + { + "first_api_call_start_time": datetime(2026, 1, 1, 12, 0, 1), + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "count these input tokens please"}], + "call_type": "acompletion", + } + ) + logging_obj.defer_async_logging = True + assert logging_obj.claim_async_success_log() + logging_obj.enqueue_deferred_logging = lambda: None + request_data = {"litellm_logging_obj": logging_obj, "metadata": {}} + await self._run( + request_data, + original_exception=HTTPException( + status_code=LITELLM_HTTP_STATUS_CLIENT_DISCONNECTED, detail="Client disconnected the request" + ), + ) + + assert request_data["response_cost"] > 0.0 + @pytest.mark.asyncio async def test_client_disconnect_before_provider_handoff_is_not_priced(self): """A caller that hung up while the request was still in pre-call hooks never reached the From dd511a2b22942d298a3c523a714a2b16ebcf6459 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Tue, 20 Oct 2026 15:19:14 -0700 Subject: [PATCH 009/235] fix(logging)!: fire one CustomLogger failure hook per async call outside the async flag list (#47124) * fix(logging): fire one CustomLogger hook per async call outside the async flag list aspeech, atranscription, aimage_generation, aimage_edit, aocr and asearch are missing from is_sync_litellm_request's flag list, so the sync success and failure handlers called a CustomLogger's sync hook on top of the async hook the async handler had already called for the same request. The sync handlers now skip a CustomLogger's log_success_event and log_failure_event when that run is paired with an async handler run (for_async_request) and the async handler gives the logger its own async hook. The async client wrapper, dispatch_failure_handlers, the proxy-only failure dispatch and the native bridge's sync failure step for an async call mark their sync failure run as paired. Resolves LIT-9486 * fix(logging): keep the sync CustomLogger hook when no async success runs The async client wrapper skips async success logging for calls carrying a fallbacks kwarg, and the native bridge skips it for internal and fallbacks calls, yet the paired sync success run still dropped a both-hooks logger's sync hook, so that logger got no success event at all. The sync success run now suppresses the sync hook only when the async success handler was actually scheduled for the call: the client wrapper passes async_handler_runs=not is_completion_with_fallbacks, and the native host records per call whether it enqueued or deferred the async success. Tests that pinned the doubled failure hook on native aocr, messages and responses calls and on proxy-rejected requests now expect the async hook alone. The integration test adds /v1/messages and /v1/responses cells and runs the key-auth rejection on every route. * fix(logging): type dispatch_success_handlers kwargs The new async_handler_runs keyword on success_handler made the untyped **kwargs that dispatch_success_handlers forwards an unknown argument for it, which grew the reportUnknownArgumentType count past its merge-base ceiling. Typing the forwarded kwargs as Any, as _success_handler_body already does, removes those diagnostics without changing what is forwarded. * test(integration): cover CustomLogger hook counts across modes, chaos and router strategies Moves the hook count rig into tests/integration/_support/hook_counts.py and extends the cells it drives: - streamed chat, Messages and Responses requests, success and failure - loggers the async handler never reaches (sync-only overrides, sync event lists only) keep one sync hook per request on every surface outside the async flag list, key-auth rejections included - a concurrent burst while the logger's destination stalls and then fails, and a burst while a worker is killed, never give a request two hooks - least-busy keeps counting a request in flight while async speech requests finish around it, and usage-based routing v2 counts an async image request's tokens once * test(integration): Datadog and LangSmith deliver one record per async request Points the built-in Datadog, LangSmith and OpenTelemetry loggers at a local sink and checks every async surface outside the flag list: one Datadog log entry and one LangSmith run per request (two on the merge base), and never two OpenTelemetry spans, which its own emit-once guard already ensured. * fix(logging): give a both-hooks logger its sync hook when the async run never reaches it The sync hook was skipped whenever the logger sat in the async callback list, so a logger the async run never reached got no hook at all: a sibling stalled past the logging worker timeout, a batch poll that skips default logging, or an async failure run cut off. The sync run now holds the hook and the async run hands it back unless it reached the logger * fix(logging): run the held sync success hooks when the proxy drops a blocked response's success A post-call guardrail block drops the parked async success, which left the sync success hook held for it unrun. Base gave a both-hooks logger that hook on these call types, so the drop now ends the pairing and the held hook runs as before * fix(logging): run held sync hooks only after every sync run of the request has closed A held hook released by the async run could run on another thread while the sync run was still walking its other callbacks. The pairing now waits until the async run has ended and every sync run has closed, and the last sync run to close runs the held hooks itself * fix(logging): end the async run when its coroutine is dropped before it starts A logging queue that drops the async success coroutine, or a deferred success that fails to schedule, left the sync hooks held for it unrun. The enqueued async run now ends when its coroutine is discarded, and the proxy ends it when firing the deferred success raises * fix(logging): leave a still-queued async run to the exit flush The discard finalizer ran at interpreter exit before the logging worker's exit flush, so a queued request could get its held sync hook and then its async hook * test: keep the router and file-stream tests in step with main Main gives each owned proxy a private Redis by default, so the router cells share the worker's Redis to read its counters, and the file-stream stub takes the hook pairing the stream now passes * fix(logging): run held sync hooks when a during_call guardrail raises or the exit flush gives up A during_call guardrail that raised after the LLM call parked its deferred success skipped the proxy's flush, and async runs the exit flush never started were never ended, so a both-hooks logger lost its sync hook in both cases * fix(logging): skip the sync failure hook when the async failure run logs to the logger An async call outside is_sync_litellm_request's flag list ran both failure handlers, so a CustomLogger with both failure hooks got both. The sync run of an async request now skips log_failure_event for a logger that overrides async_log_failure_event and is registered for async failures. The four sync runs of async failures (dispatch_failure_handlers, the @client async wrapper, the proxy-only failure thread and the Rust host) pass for_async_request=True. Drops the HookPairing fallback. Successes are left to #47192, which already gives such a logger only async_log_success_event. --------- Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 18 +- litellm/proxy/utils.py | 1 + .../host/callbacks_legacy_python.py | 21 +- litellm/utils.py | 2 +- .../integration/_support/hook_count_probe.py | 91 +++++ tests/integration/_support/hook_counts.py | 329 ++++++++++++++++++ .../test_custom_logger_hook_counts.py | 96 +++++ .../test_custom_logger_hook_counts_chaos.py | 181 ++++++++++ .../test_custom_logger_hook_counts_modes.py | 95 +++++ ...stom_logger_hook_counts_post_call_block.py | 75 ++++ .../test_custom_logger_hook_counts_vendors.py | 128 +++++++ ...er_strategies_count_async_requests_once.py | 214 ++++++++++++ .../messages/test_callbacks.py | 2 +- tests/test_litellm_rust/ocr/test_callbacks.py | 6 +- tests/test_litellm_rust/ocr/test_lifecycle.py | 16 +- .../responses/test_callbacks.py | 2 +- .../test_litellm_logging.py | 151 +++++++- tests/unit/proxy/test_proxy_reject_logging.py | 2 +- tests/unit/proxy/test_proxy_utils.py | 52 +++ .../host/test_callbacks_legacy_python.py | 70 ++++ 20 files changed, 1532 insertions(+), 20 deletions(-) create mode 100644 tests/integration/_support/hook_count_probe.py create mode 100644 tests/integration/_support/hook_counts.py create mode 100644 tests/integration/observability/test_custom_logger_hook_counts.py create mode 100644 tests/integration/observability/test_custom_logger_hook_counts_chaos.py create mode 100644 tests/integration/observability/test_custom_logger_hook_counts_modes.py create mode 100644 tests/integration/observability/test_custom_logger_hook_counts_post_call_block.py create mode 100644 tests/integration/observability/test_custom_logger_hook_counts_vendors.py create mode 100644 tests/integration/routing/test_router_strategies_count_async_requests_once.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 069dcffd21a..93b45be4922 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2466,7 +2466,7 @@ class Logging(LiteLLMLoggingBaseClass): await self.async_failure_handler(exception, traceback_exception) finally: if self._should_run_sync_failure_callbacks_for_async_calls(): - executor.submit(self.failure_handler, exception, traceback_exception) + executor.submit(self.failure_handler, exception, traceback_exception, for_async_request=True) def should_run_logging( self, @@ -4090,15 +4090,21 @@ class Logging(LiteLLMLoggingBaseClass): traceback_exception: str, start_time: datetime.datetime | None = None, end_time: datetime.datetime | None = None, + *, + for_async_request: bool = False, ) -> None: """Restores trace_id/session_id contextvars once this attempt's own failure - logging (including any nested calls its callbacks trigger) is fully done.""" + logging (including any nested calls its callbacks trigger) is fully done. + + ``for_async_request`` marks the sync run of a failure whose async run also logs it, so a + CustomLogger that gets ``async_log_failure_event`` from that async run skips its sync hook.""" try: return self._failure_handler_body( exception=exception, traceback_exception=traceback_exception, start_time=start_time, end_time=end_time, + for_async_request=for_async_request, ) finally: self._restore_correlation_context() @@ -4109,6 +4115,7 @@ class Logging(LiteLLMLoggingBaseClass): traceback_exception: str, start_time: datetime.datetime | None = None, end_time: datetime.datetime | None = None, + for_async_request: bool = False, ) -> None: verbose_logger.debug("Logging Details LiteLLM-Failure Call: %s", litellm.failure_callback) if not self.should_run_logging(event_type="sync_failure"): # prevent double logging @@ -4201,6 +4208,7 @@ class Logging(LiteLLMLoggingBaseClass): isinstance(callback, CustomLogger) and is_sync_request and self.call_type != CallTypes.pass_through.value + and not (for_async_request and self._async_handler_logs_to(callback)) ): # custom logger class callback.log_failure_event( start_time=start_time, @@ -4563,6 +4571,12 @@ class Logging(LiteLLMLoggingBaseClass): cb_name: Final = self._get_callback_name(cb) return any(prefix in cb_name for prefix in INTERNAL_PREFIXES) + def _async_handler_logs_to(self, logger: CustomLogger) -> bool: + """True when the async failure run of this call hands ``logger`` its own ``async_log_failure_event``.""" + if type(logger).async_log_failure_event is CustomLogger.async_log_failure_event: # pyright: ignore[reportUnknownMemberType] # CustomLogger's hook signatures are untyped + return False + return logger in (self.dynamic_async_failure_callbacks or []) or logger in litellm._async_failure_callback # pyright: ignore[reportUnknownMemberType, reportPrivateUsage] # callback lists are untyped and have no public accessor + def _remove_internal_custom_logger_callbacks(self, callbacks: list) -> list: """ Removes internal custom logger callbacks from the list. diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index aca222c31ef..c5983077d4c 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3865,6 +3865,7 @@ class ProxyLogging: original_exception, traceback_str, ), + kwargs={"for_async_request": True}, daemon=True, ).start() diff --git a/litellm/rust_bridge/host/callbacks_legacy_python.py b/litellm/rust_bridge/host/callbacks_legacy_python.py index a82e734b47d..ae913882968 100644 --- a/litellm/rust_bridge/host/callbacks_legacy_python.py +++ b/litellm/rust_bridge/host/callbacks_legacy_python.py @@ -11,6 +11,7 @@ import contextvars import datetime import traceback import uuid +import weakref from collections.abc import Awaitable, Coroutine, Mapping from dataclasses import dataclass from typing import ( @@ -61,9 +62,21 @@ def setup( } supplied: Final = arguments.get("litellm_logging_obj") if isinstance(supplied, Logging): - return _claim_budget_reservation(CallSetup(supplied, arguments), asynchronous) + return _begin_call(CallSetup(supplied, arguments), asynchronous) logger, prepared = function_setup(call_type, Rules(), start_time, *args, is_async_call=asynchronous, **arguments) - return _claim_budget_reservation(CallSetup(logger, prepared), asynchronous) + return _begin_call(CallSetup(logger, prepared), asynchronous) + + +# Loggers whose current native call is async: their sync failure step is followed by the async one +_ASYNC_CALLS: Final[weakref.WeakSet[Logging]] = weakref.WeakSet() + + +def _begin_call(call_setup: CallSetup, asynchronous: bool) -> CallSetup: + if asynchronous: + _ASYNC_CALLS.add(call_setup.logger) + else: + _ASYNC_CALLS.discard(call_setup.logger) + return _claim_budget_reservation(call_setup, asynchronous) def _claim_budget_reservation(call_setup: CallSetup, asynchronous: bool) -> CallSetup: @@ -135,6 +148,8 @@ class LoggingSurface(Protocol): traceback_exception: str, start_time: datetime.datetime | None = None, end_time: datetime.datetime | None = None, + *, + for_async_request: bool = False, ) -> None: ... def async_failure_handler( @@ -261,7 +276,7 @@ def failure_handler( if not is_internal_call(): unbind_budget_reservation_from_callbacks(logger.litellm_params) return logger.async_failure_handler(error, trace, start, end) - logger.failure_handler(error, trace, start, end) + logger.failure_handler(error, trace, start, end, for_async_request=logger in _ASYNC_CALLS) return None diff --git a/litellm/utils.py b/litellm/utils.py index 987139b7530..a9079f470c7 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2371,7 +2371,7 @@ def client(original_function): unbind_budget_reservation_from_callbacks(logging_obj.litellm_params) try: logging_obj.failure_handler( - e, traceback_exception, start_time, end_time + e, traceback_exception, start_time, end_time, for_async_request=True ) # DO NOT MAKE THREADED - router retry fallback relies on this! except Exception as e: raise e diff --git a/tests/integration/_support/hook_count_probe.py b/tests/integration/_support/hook_count_probe.py new file mode 100644 index 00000000000..0d4056e8fb9 --- /dev/null +++ b/tests/integration/_support/hook_count_probe.py @@ -0,0 +1,91 @@ +import json +import os +from collections.abc import Mapping +from datetime import datetime +from typing import Final + +import httpx +from fastapi import HTTPException + +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_logger import CustomLogger + + +def _record(hook: str, kwargs: Mapping[str, object]) -> bytes: + return json.dumps( + {"hook": hook, "litellm_call_id": kwargs.get("litellm_call_id"), "call_type": kwargs.get("call_type")} + ).encode() + + +def _send(hook: str, kwargs: Mapping[str, object]) -> None: + with httpx.Client(trust_env=False) as client: + client.post(os.environ["HOOK_COUNT_PROBE_URL"], content=_record(hook, kwargs)).raise_for_status() + + +async def _send_async(hook: str, kwargs: Mapping[str, object]) -> None: + async with httpx.AsyncClient(trust_env=False) as client: + response: Final = await client.post(os.environ["HOOK_COUNT_PROBE_URL"], content=_record(hook, kwargs)) + response.raise_for_status() + + +class HookCountProbe(CustomLogger): + """Reports every sync and async success and failure hook it receives, one record per call.""" + + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + _send("sync_success", kwargs) + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + await _send_async("async_success", kwargs) + + def log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + _send("sync_failure", kwargs) + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + await _send_async("async_failure", kwargs) + + +class SyncOnlyHookCountProbe(CustomLogger): + """Overrides only the sync success and failure hooks.""" + + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + _send("sync_success", kwargs) + + def log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + _send("sync_failure", kwargs) + + +class BlockingPostCallGuardrail(CustomGuardrail): + """Rejects every response with a 400 after the provider answered.""" + + async def async_post_call_success_hook( + self, data: Mapping[str, object], user_api_key_dict: object, response: object + ) -> None: + raise HTTPException(status_code=400, detail="blocked by the hook-count guardrail") + + +probe: Final = HookCountProbe() +sync_only_probe: Final = SyncOnlyHookCountProbe() + + +def sync_success_function( + kwargs: Mapping[str, object], completion_response: object, start_time: datetime, end_time: datetime +) -> None: + _send("function_success", kwargs) + + +def sync_failure_function( + kwargs: Mapping[str, object], completion_response: object, start_time: datetime, end_time: datetime +) -> None: + _send("function_failure", kwargs) diff --git a/tests/integration/_support/hook_counts.py b/tests/integration/_support/hook_counts.py new file mode 100644 index 00000000000..6b7f482bd85 --- /dev/null +++ b/tests/integration/_support/hook_counts.py @@ -0,0 +1,329 @@ +"""Scripted provider, proxy config and request helpers shared by the CustomLogger hook count tests.""" + +import base64 +import json +import threading +from collections.abc import Callable, Mapping +from dataclasses import dataclass, field +from pathlib import Path +from typing import Final + +import httpx +import yaml +from integration._support.client import Gateway, held +from integration._support.wire import Reply, Request, Wire + +WAV_BYTES: Final = ( + b"RIFF\x30\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00" + b"\x40\x1f\x00\x00\x40\x1f\x00\x00\x01\x00\x08\x00data\x08\x00\x00\x00" + b"\x00\x01\x02\x03\x04\x05\x06\x07" +) +PNG_BYTES: Final = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00" + b"\x1f\x15\xc4\x89\x00\x00\x00\rIDAT\x08\xd7c\xf8\xcf\xc0\xf0\x1f\x00\x05\x00\x01\xff" + b"\x89\x99=\x1d\x00\x00\x00\x00IEND\xaeB`\x82" +) +_IMAGE_REPLY: Final = json.dumps({"created": 1700000000, "data": [{"b64_json": base64.b64encode(PNG_BYTES).decode()}]}) +_CHAT_REPLY: Final = json.dumps( + { + "id": "chatcmpl-hook-count", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4}, + } +) +_EMBEDDING_REPLY: Final = json.dumps( + { + "object": "list", + "model": "text-embedding-3-small", + "data": [{"object": "embedding", "index": 0, "embedding": [0.25, 0.5]}], + "usage": {"prompt_tokens": 3, "total_tokens": 3}, + } +) +_MESSAGE_REPLY: Final = json.dumps( + { + "id": "msg_hook_count", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "ok"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 3, "output_tokens": 1}, + } +) +_RESPONSE_REPLY: Final = json.dumps( + { + "id": "resp_hook_count", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": "msg_hook_count", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "ok", "annotations": []}], + } + ], + "usage": {"input_tokens": 3, "output_tokens": 1, "total_tokens": 4}, + } +) +_OCR_REPLY: Final = json.dumps( + { + "pages": [ + {"index": 0, "markdown": "page", "images": [], "dimensions": {"dpi": 200, "height": 1100, "width": 850}} + ], + "model": "mistral-ocr-latest", + "usage_info": {"pages_processed": 1, "doc_size_bytes": 64}, + "document_annotation": None, + } +) +_SEARCH_REPLY: Final = json.dumps( + {"results": [{"title": "hook count", "url": "https://result.invalid/a", "snippet": "hit"}]} +) +_CHAT_FRAMES: Final = ( + 'data: {"id":"chatcmpl-hook-count","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini",' + '"choices":[{"index":0,"delta":{"role":"assistant","content":"ok"},"finish_reason":null}]}', + 'data: {"id":"chatcmpl-hook-count","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini",' + '"choices":[{"index":0,"delta":{},"finish_reason":"stop"}],' + '"usage":{"prompt_tokens":3,"completion_tokens":1,"total_tokens":4}}', + "data: [DONE]", +) +_MESSAGE_FRAMES: Final = ( + "event: message_start\n" + 'data: {"type":"message_start","message":{"id":"msg_hook_count","type":"message","role":"assistant",' + '"model":"claude-haiku-4-5","content":[],"stop_reason":null,"stop_sequence":null,' + '"usage":{"input_tokens":3,"output_tokens":0}}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + "event: content_block_delta\n" + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"ok"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + "event: message_delta\n" + 'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":1}}', + 'event: message_stop\ndata: {"type":"message_stop"}', +) +_RESPONSE_FRAMES: Final = ( + "event: response.created\n" + 'data: {"type":"response.created","sequence_number":0,"response":{"id":"resp_hook_count","object":"response",' + '"created_at":1,"status":"in_progress","model":"gpt-4o-mini","output":[],"usage":null}}', + "event: response.output_text.delta\n" + 'data: {"type":"response.output_text.delta","sequence_number":1,"item_id":"msg_hook_count","output_index":0,' + '"content_index":0,"delta":"ok"}', + "event: response.completed\n" + 'data: {"type":"response.completed","sequence_number":2,"response":{"id":"resp_hook_count","object":"response",' + '"created_at":1,"status":"completed","model":"gpt-4o-mini","output":[{"type":"message","id":"msg_hook_count",' + '"status":"completed","role":"assistant","content":[{"type":"output_text","text":"ok","annotations":[]}]}],' + '"usage":{"input_tokens":3,"output_tokens":1,"total_tokens":4}}}', +) +_STREAM_FRAMES: Final = { + "/v1/chat/completions": _CHAT_FRAMES, + "/v1/messages": _MESSAGE_FRAMES, + "/v1/responses": _RESPONSE_FRAMES, +} +SURFACE_CALL_TYPES: Final = { + "chat": "acompletion", + "embedding": "aembedding", + "messages": "anthropic_messages", + "responses": "aresponses", + "speech": "aspeech", + "transcription": "atranscription", + "image_generation": "aimage_generation", + "image_edit": "aimage_edit", + "ocr": "aocr", + "search": "asearch", +} +IN_THE_ASYNC_FLAG_LIST: Final = frozenset({"chat", "embedding", "messages", "responses"}) +STREAMING_SURFACES: Final = ("chat", "messages", "responses") + + +def _streams(request: Request) -> bool: + if request.headers.get("content-type", "").startswith("application/json"): + return json.loads(request.body or b"{}").get("stream") is True + return False + + +def provider(request: Request) -> Reply: + """Answers every surface the hook count tests drive; any target under /fail/ is a provider 400.""" + if request.method == "GET": + return Reply(body=b'{"object":"list","data":[]}') + if request.target.startswith("/fail/"): + return Reply( + status=400, + body=b'{"error":{"message":"scripted provider rejection","type":"invalid_request_error","code":null}}', + ) + if request.target in _STREAM_FRAMES and _streams(request): + return Reply( + chunks=tuple(f"{frame}\n\n".encode() for frame in _STREAM_FRAMES[request.target]), + content_type="text/event-stream", + ) + replies: Final[dict[str, Callable[[], Reply]]] = { + "/v1/chat/completions": lambda: Reply(body=_CHAT_REPLY.encode()), + "/v1/embeddings": lambda: Reply(body=_EMBEDDING_REPLY.encode()), + "/v1/messages": lambda: Reply(body=_MESSAGE_REPLY.encode()), + "/v1/responses": lambda: Reply(body=_RESPONSE_REPLY.encode()), + "/v1/audio/speech": lambda: Reply(body=WAV_BYTES, content_type="audio/wav"), + "/v1/audio/transcriptions": lambda: Reply(body=b'{"text":"hook count transcript"}'), + "/v1/images/generations": lambda: Reply(body=_IMAGE_REPLY.encode()), + "/v1/images/edits": lambda: Reply(body=_IMAGE_REPLY.encode()), + "/v1/ocr": lambda: Reply(body=_OCR_REPLY.encode()), + "/perplexity/search": lambda: Reply(body=_SEARCH_REPLY.encode()), + } + return replies[request.target]() + + +@dataclass +class HookRecords: + """Every hook record the probe reported, kept across drains so cells can read their own call id.""" + + wire: Wire + function_callbacks: bool + seen: list[dict[str, str]] = field(default_factory=list) + lock: threading.Lock = field(default_factory=threading.Lock) + + def drained(self) -> tuple[dict[str, str], ...]: + with self.lock: + self.seen.extend(json.loads(request.body) for request in self.wire.drain()) + return tuple(self.seen) + + def for_call(self, call_id: str) -> tuple[dict[str, str], ...]: + return tuple(record for record in self.drained() if record["litellm_call_id"] == call_id) + + +def _model(name: str, model: str, api_base: str, **extra: object) -> dict[str, object]: + return { + "model_name": name, + "litellm_params": {"model": model, "api_base": api_base, "api_key": "synthetic-provider-key"}, + **extra, + } + + +def write_hook_count_config( + path: Path, + provider_url: str, + litellm_settings: Mapping[str, object], + guardrails: tuple[Mapping[str, object], ...] = (), +) -> Path: + """One success and one failure deployment per surface, all answered by `provider`.""" + models: Final = [] + tools: Final = [] + for outcome, prefix in (("success", ""), ("failure", "/fail")): + base: str = provider_url + prefix + models.extend( + ( + _model(f"hooks-chat-{outcome}", "openai/gpt-4o-mini", base + "/v1"), + _model(f"hooks-embedding-{outcome}", "openai/text-embedding-3-small", base + "/v1"), + _model(f"hooks-messages-{outcome}", "anthropic/claude-haiku-4-5", base), + _model(f"hooks-responses-{outcome}", "openai/gpt-4o-mini", base + "/v1"), + _model(f"hooks-speech-{outcome}", "openai/gpt-4o-mini-tts", base + "/v1"), + _model(f"hooks-transcription-{outcome}", "openai/whisper-1", base + "/v1"), + _model(f"hooks-image_generation-{outcome}", "openai/gpt-image-1", base + "/v1"), + _model(f"hooks-image_edit-{outcome}", "openai/gpt-image-1", base + "/v1"), + _model(f"hooks-ocr-{outcome}", "mistral/mistral-ocr-latest", base + "/v1", model_info={"mode": "ocr"}), + ) + ) + tools.append( + { + "search_tool_name": f"hooks-search-{outcome}", + "litellm_params": { + "search_provider": "perplexity", + "api_key": "synthetic-provider-key", + "api_base": base + "/perplexity", + }, + } + ) + path.write_text( + yaml.safe_dump( + { + "model_list": models, + "search_tools": tools, + "litellm_settings": dict(litellm_settings), + **({"guardrails": [dict(guardrail) for guardrail in guardrails]} if guardrails else {}), + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_model_info_refresh": True, + }, + "router_settings": {"num_retries": 0}, + } + ) + ) + return path + + +def send( + proxy: Gateway, surface: str, outcome: str, call_id: str, *, key: str | None = None, stream: bool = False +) -> httpx.Response: + model: Final = f"hooks-{surface}-{outcome}" + nonce: Final = f"hook-count-{call_id}" + headers: Final = {"Authorization": f"Bearer {key or proxy.key}", "x-litellm-call-id": call_id} + if surface == "transcription": + return proxy.client.post( + "/v1/audio/transcriptions", + data={"model": model, "prompt": nonce}, + files={"file": ("a.wav", WAV_BYTES + call_id.encode(), "audio/wav")}, + headers=headers, + ) + if surface == "image_edit": + return proxy.client.post( + "/v1/images/edits", + data={"model": model, "prompt": nonce}, + files={"image": ("a.png", PNG_BYTES, "image/png")}, + headers=headers, + ) + streaming: Final[dict[str, object]] = {"stream": True} if stream else {} + routes: Final[dict[str, tuple[str, dict[str, object]]]] = { + "chat": ( + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": nonce}], **streaming}, + ), + "embedding": ("/v1/embeddings", {"model": model, "input": nonce}), + "messages": ( + "/v1/messages", + {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": nonce}], **streaming}, + ), + "responses": ("/v1/responses", {"model": model, "input": nonce, **streaming}), + "speech": ("/v1/audio/speech", {"model": model, "input": nonce, "voice": "alloy", "response_format": "wav"}), + "image_generation": ("/v1/images/generations", {"model": model, "prompt": nonce, "n": 1}), + "ocr": ( + "/v1/ocr", + { + "model": model, + "document": { + "type": "document_url", + "document_url": "data:application/pdf;base64," + base64.b64encode(nonce.encode()).decode(), + }, + }, + ), + "search": ("/v1/search", {"search_tool_name": f"hooks-search-{outcome}", "query": nonce}), + } + path, body = routes[surface] + return proxy.client.post(path, json=body, headers=headers) + + +def counts_function_hooks(records: HookRecords, surface: str) -> bool: + return records.function_callbacks and surface not in IN_THE_ASYNC_FLAG_LIST + + +def probe_hooks(records: HookRecords, surface: str, found: tuple[dict[str, str], ...]) -> list[str]: + counted: Final = counts_function_hooks(records, surface) + return sorted(record["hook"] for record in found if counted or not record["hook"].startswith("function_")) + + +def one_async_hook(records: HookRecords, surface: str, outcome: str) -> list[str]: + function_hook: Final = (f"function_{outcome}",) if counts_function_hooks(records, surface) else () + return sorted((f"async_{outcome}", *function_hook)) + + +def assert_hooks(records: HookRecords, surface: str, call_id: str, expected: list[str]) -> None: + settled: Final = held( + lambda: records.for_call(call_id), + lambda found: probe_hooks(records, surface, found) == expected, + holding=3, + seconds=20, + ) + assert {record["call_type"] for record in settled} == {SURFACE_CALL_TYPES[surface]}, settled diff --git a/tests/integration/observability/test_custom_logger_hook_counts.py b/tests/integration/observability/test_custom_logger_hook_counts.py new file mode 100644 index 00000000000..420ac2a5fcb --- /dev/null +++ b/tests/integration/observability/test_custom_logger_hook_counts.py @@ -0,0 +1,96 @@ +import uuid +from collections.abc import Iterator +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.hook_counts import ( + STREAMING_SURFACES, + SURFACE_CALL_TYPES, + HookRecords, + assert_hooks, + one_async_hook, + provider, + send, + write_hook_count_config, +) +from integration._support.process import owned_proxy +from integration._support.wire import Reply, wire_server + +pytestmark = pytest.mark.timeout(240) + + +@pytest.fixture(scope="module", params=["cache", "cache_and_sync_callback"]) +def hook_proxy( + request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[tuple[Gateway, HookRecords]]: + tmp_path: Final = tmp_path_factory.mktemp("custom-logger-hook-counts") + legacy_callbacks: Final = ( + { + "success_callback": ["integration._support.hook_count_probe.sync_success_function"], + "failure_callback": ["integration._support.hook_count_probe.sync_failure_function"], + } + if request.param == "cache_and_sync_callback" + else {} + ) + with ( + wire_server(provider) as upstream, + wire_server(lambda request: Reply()) as sink, + gateway_from_environment() as rig, + ): + config: Final = write_hook_count_config( + tmp_path / "hook-counts.yaml", + upstream.url, + { + "callbacks": ["integration._support.hook_count_probe.probe"], + **legacy_callbacks, + "cache": True, + "cache_params": {"type": "redis", "host": "os.environ/REDIS_HOST", "port": "os.environ/REDIS_PORT"}, + }, + ) + with owned_proxy( + rig, tmp_path, {"HOOK_COUNT_PROBE_URL": sink.url, "STORE_MODEL_IN_DB": "False"}, config=config, workers=2 + ) as proxy: + yield proxy, HookRecords(sink, function_callbacks=request.param == "cache_and_sync_callback") + + +@pytest.mark.parametrize("outcome", ["success", "failure"]) +@pytest.mark.parametrize("surface", sorted(SURFACE_CALL_TYPES)) +def test_custom_logger_gets_one_async_hook_per_request_with_the_response_cache_on( + hook_proxy: tuple[Gateway, HookRecords], surface: str, outcome: str +) -> None: + proxy, records = hook_proxy + call_id: Final = str(uuid.uuid4()) + response: Final = send(proxy, surface, outcome, call_id) + assert response.status_code == (200 if outcome == "success" else 400), response.text + assert_hooks(records, surface, call_id, one_async_hook(records, surface, outcome)) + + +@pytest.mark.parametrize("outcome", ["success", "failure"]) +@pytest.mark.parametrize("surface", STREAMING_SURFACES) +def test_custom_logger_gets_one_async_hook_per_streamed_request( + hook_proxy: tuple[Gateway, HookRecords], surface: str, outcome: str +) -> None: + proxy, records = hook_proxy + call_id: Final = str(uuid.uuid4()) + response: Final = send(proxy, surface, outcome, call_id, stream=True) + assert response.status_code == (200 if outcome == "success" else 400), response.text + if outcome == "success": + assert "ok" in response.text, response.text + assert_hooks(records, surface, call_id, one_async_hook(records, surface, outcome)) + + +@pytest.mark.parametrize("surface", sorted(SURFACE_CALL_TYPES)) +def test_custom_logger_gets_one_async_hook_per_request_rejected_by_key_auth( + hook_proxy: tuple[Gateway, HookRecords], surface: str +) -> None: + proxy, records = hook_proxy + known_calls: Final = {record["litellm_call_id"] for record in records.drained()} + response: Final = send(proxy, surface, "success", str(uuid.uuid4()), key="sk-not-a-virtual-key") + assert response.status_code == 401, response.text + + def new_calls(found: tuple[dict[str, str], ...]) -> set[str]: + return {record["litellm_call_id"] for record in found if record["litellm_call_id"] not in known_calls} + + (rejected_call,) = eventually(lambda: new_calls(records.drained()), bool, seconds=20) + assert_hooks(records, surface, rejected_call, one_async_hook(records, surface, "failure")) diff --git a/tests/integration/observability/test_custom_logger_hook_counts_chaos.py b/tests/integration/observability/test_custom_logger_hook_counts_chaos.py new file mode 100644 index 00000000000..7b19ef3b1ff --- /dev/null +++ b/tests/integration/observability/test_custom_logger_hook_counts_chaos.py @@ -0,0 +1,181 @@ +"""One hook per request under a concurrent burst while the logger's destination stalls and fails, and while a +worker is killed.""" + +import threading +import time +import uuid +from collections.abc import Callable +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, eventually, gateway_from_environment, held +from integration._support.hook_counts import ( + STREAMING_SURFACES, + SURFACE_CALL_TYPES, + HookRecords, + assert_hooks, + one_async_hook, + probe_hooks, + provider, + send, + write_hook_count_config, +) +from integration._support.process import graceful_stop_seconds, owned_proxy, owned_proxy_process +from integration._support.wire import Reply, Request, wire_server + +pytestmark = pytest.mark.timeout(2 * graceful_stop_seconds() + 300) + +RecordProperty = Callable[[str, object], None] + +_SETTINGS: Final = {"callbacks": ["integration._support.hook_count_probe.probe"]} +_HOLD_SECONDS: Final = 2.0 + + +@dataclass(frozen=True) +class _Call: + surface: str + outcome: str + stream: bool + call_id: str + + +@dataclass(frozen=True) +class _Outcome: + call: _Call + status: int | None + error: str | None + + +def _burst(rounds: int) -> tuple[_Call, ...]: + shapes: Final = ( + *((surface, outcome, False) for surface in sorted(SURFACE_CALL_TYPES) for outcome in ("success", "failure")), + *((surface, "success", True) for surface in STREAMING_SURFACES), + ) + return tuple( + _Call(surface, outcome, stream, str(uuid.uuid4())) for _ in range(rounds) for surface, outcome, stream in shapes + ) + + +def _attempt(proxy: Gateway, call: _Call) -> _Outcome: + try: + response: Final = send(proxy, call.surface, call.outcome, call.call_id, stream=call.stream) + except httpx.HTTPError as error: + return _Outcome(call, None, repr(error)) + return _Outcome(call, response.status_code, None) + + +def _expected_status(call: _Call) -> int: + return 200 if call.outcome == "success" else 400 + + +def _done(futures: tuple[Future[_Outcome], ...]) -> int: + return sum(future.done() for future in futures) + + +class _Destination: + """The probe's sink: answers at once, holds each report until released, or refuses it with a 503.""" + + def __init__(self) -> None: + self.mode = "ok" + self.release = threading.Event() + + def __call__(self, request: Request) -> Reply: + if self.mode == "hold": + self.release.wait(timeout=_HOLD_SECONDS + 1) + if self.mode == "fail": + return Reply(status=503, body=b'{"error":"destination unavailable"}') + return Reply() + + +def test_logger_destination_stalling_then_failing_mid_burst_gets_one_hook_per_request( + tmp_path: Path, record_property: RecordProperty +) -> None: + destination: Final = _Destination() + with ( + wire_server(provider) as upstream, + wire_server(destination) as sink, + gateway_from_environment() as rig, + ): + config: Final = write_hook_count_config(tmp_path / "hook-count-chaos.yaml", upstream.url, _SETTINGS) + with owned_proxy( + rig, tmp_path, {"HOOK_COUNT_PROBE_URL": sink.url, "STORE_MODEL_IN_DB": "False"}, config=config, workers=2 + ) as proxy: + records: Final = HookRecords(sink, function_callbacks=False) + calls: Final = _burst(2) + destination.mode = "hold" + with ThreadPoolExecutor(max_workers=len(calls)) as pool: + futures: Final = tuple(pool.submit(_attempt, proxy, call) for call in calls) + eventually(lambda: _done(futures), lambda done: done >= 1, seconds=60) + held_liveliness: Final = proxy.client.get("/health/liveliness").status_code + time.sleep(_HOLD_SECONDS) + destination.mode = "fail" + destination.release.set() + time.sleep(_HOLD_SECONDS) + destination.mode = "ok" + outcomes: Final = tuple(future.result(timeout=180) for future in futures) + record_property("burst_size", len(calls)) + assert held_liveliness == 200, held_liveliness + assert [outcome for outcome in outcomes if outcome.status != _expected_status(outcome.call)] == [] + for call in calls: + assert_hooks(records, call.surface, call.call_id, one_async_hook(records, call.surface, call.outcome)) + + +def test_worker_killed_mid_burst_leaves_at_most_one_hook_per_request( + tmp_path: Path, record_property: RecordProperty +) -> None: + with ( + wire_server(provider) as upstream, + wire_server(lambda request: Reply()) as sink, + gateway_from_environment() as rig, + ): + config: Final = write_hook_count_config(tmp_path / "hook-count-kill.yaml", upstream.url, _SETTINGS) + with owned_proxy_process( + rig, tmp_path, {"HOOK_COUNT_PROBE_URL": sink.url, "STORE_MODEL_IN_DB": "False"}, config=config, workers=2 + ) as owned: + proxy: Final = owned.gateway + records: Final = HookRecords(sink, function_callbacks=False) + victim: Final = min(eventually(owned.worker_pids, lambda found: len(found) == 2, seconds=60)) + calls: Final = _burst(2) + with ThreadPoolExecutor(max_workers=len(calls)) as pool: + futures: Final = tuple(pool.submit(_attempt, proxy, call) for call in calls) + eventually(lambda: _done(futures), lambda done: done >= 1, seconds=60) + with owned.replacing_worker(victim) as kill: + kill() + done_at_kill: Final = _done(futures) + survived: Final = eventually( + lambda: _attempt(proxy, _Call("speech", "success", False, str(uuid.uuid4()))), + lambda outcome: outcome.status == 200, + seconds=30, + ) + outcomes: Final = tuple(future.result(timeout=180) for future in futures) + respawned: Final = _Call("speech", "success", False, str(uuid.uuid4())) + after_respawn: Final = _attempt(proxy, respawned) + completed: Final = tuple(outcome for outcome in outcomes if outcome.status is not None) + interrupted: Final = tuple(outcome for outcome in outcomes if outcome.status is None) + assert after_respawn.status == 200, after_respawn + assert [outcome for outcome in completed if outcome.status != _expected_status(outcome.call)] == [] + for call in (survived.call, respawned): + assert_hooks(records, call.surface, call.call_id, one_async_hook(records, call.surface, call.outcome)) + burst_hooks: Final = held( + lambda: _hooks_of(records, calls), + lambda found: all( + hooks in ([], [f"async_{call.outcome}"]) for call, hooks in zip(calls, found, strict=True) + ), + holding=3, + seconds=20, + ) + record_property("burst_size", len(calls)) + record_property("killed_worker_pid", victim) + record_property("done_at_kill", done_at_kill) + record_property("interrupted_calls", str(tuple(outcome.error for outcome in interrupted))) + record_property("burst_calls_without_a_hook", sum(not hooks for hooks in burst_hooks)) + + +def _hooks_of(records: HookRecords, calls: tuple[_Call, ...]) -> tuple[list[str], ...]: + """A burst call the killed worker served may lose its logging with the worker, whether or not its response + arrived first, so a burst call gets its one async hook or none, never two.""" + return tuple(probe_hooks(records, call.surface, records.for_call(call.call_id)) for call in calls) diff --git a/tests/integration/observability/test_custom_logger_hook_counts_modes.py b/tests/integration/observability/test_custom_logger_hook_counts_modes.py new file mode 100644 index 00000000000..e143559040e --- /dev/null +++ b/tests/integration/observability/test_custom_logger_hook_counts_modes.py @@ -0,0 +1,95 @@ +"""A logger the async handler never reaches keeps its sync hook on async calls outside the async flag list. + +A logger with both hooks registered only under `success_callback` and `failure_callback` is never reached by the async +failure handler, so it keeps its sync failure hook. The async success handler does reach it, and gives it its async +success hook instead of the sync one. + +The sync success handler only runs for an async call when a legacy sync callback is configured, so every +registration here carries the legacy function callbacks too.""" + +import uuid +from collections.abc import Iterator +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.hook_counts import ( + IN_THE_ASYNC_FLAG_LIST, + SURFACE_CALL_TYPES, + HookRecords, + assert_hooks, + provider, + send, + write_hook_count_config, +) +from integration._support.process import owned_proxy +from integration._support.wire import Reply, wire_server + +pytestmark = pytest.mark.timeout(240) + +_PROBES: Final = "integration._support.hook_count_probe" +_REGISTRATIONS: Final[dict[str, dict[str, object]]] = { + "sync_only_logger": { + "callbacks": [f"{_PROBES}.sync_only_probe"], + "success_callback": [f"{_PROBES}.sync_success_function"], + "failure_callback": [f"{_PROBES}.sync_failure_function"], + }, + "sync_event_lists": { + "success_callback": [f"{_PROBES}.probe", f"{_PROBES}.sync_success_function"], + "failure_callback": [f"{_PROBES}.probe", f"{_PROBES}.sync_failure_function"], + }, +} +_OUTSIDE_THE_ASYNC_FLAG_LIST: Final = sorted(SURFACE_CALL_TYPES.keys() - IN_THE_ASYNC_FLAG_LIST) + + +@pytest.fixture(scope="module", params=sorted(_REGISTRATIONS)) +def sync_hook_proxy( + request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[tuple[Gateway, HookRecords, str]]: + tmp_path: Final = tmp_path_factory.mktemp("custom-logger-hook-count-modes") + with ( + wire_server(provider) as upstream, + wire_server(lambda request: Reply()) as sink, + gateway_from_environment() as rig, + ): + config: Final = write_hook_count_config( + tmp_path / "hook-count-modes.yaml", upstream.url, _REGISTRATIONS[request.param] + ) + with owned_proxy( + rig, tmp_path, {"HOOK_COUNT_PROBE_URL": sink.url, "STORE_MODEL_IN_DB": "False"}, config=config, workers=2 + ) as proxy: + yield proxy, HookRecords(sink, function_callbacks=True), request.param + + +@pytest.mark.parametrize("outcome", ["success", "failure"]) +@pytest.mark.parametrize("surface", _OUTSIDE_THE_ASYNC_FLAG_LIST) +def test_logger_the_async_handler_skips_gets_one_sync_hook_per_request( + sync_hook_proxy: tuple[Gateway, HookRecords, str], surface: str, outcome: str +) -> None: + proxy, records, registration = sync_hook_proxy + reached_by_the_async_success: Final = registration == "sync_event_lists" and outcome == "success" + call_id: Final = str(uuid.uuid4()) + response: Final = send(proxy, surface, outcome, call_id) + assert response.status_code == (200 if outcome == "success" else 400), response.text + assert_hooks( + records, + surface, + call_id, + sorted([f"function_{outcome}", f"async_{outcome}" if reached_by_the_async_success else f"sync_{outcome}"]), + ) + + +@pytest.mark.parametrize("surface", _OUTSIDE_THE_ASYNC_FLAG_LIST) +def test_logger_the_async_handler_skips_gets_one_sync_hook_per_request_rejected_by_key_auth( + sync_hook_proxy: tuple[Gateway, HookRecords, str], surface: str +) -> None: + proxy, records, _ = sync_hook_proxy + known_calls: Final = {record["litellm_call_id"] for record in records.drained()} + response: Final = send(proxy, surface, "success", str(uuid.uuid4()), key="sk-not-a-virtual-key") + assert response.status_code == 401, response.text + + def new_calls(found: tuple[dict[str, str], ...]) -> set[str]: + return {record["litellm_call_id"] for record in found if record["litellm_call_id"] not in known_calls} + + (rejected_call,) = eventually(lambda: new_calls(records.drained()), bool, seconds=20) + assert_hooks(records, surface, rejected_call, ["function_failure", "sync_failure"]) diff --git a/tests/integration/observability/test_custom_logger_hook_counts_post_call_block.py b/tests/integration/observability/test_custom_logger_hook_counts_post_call_block.py new file mode 100644 index 00000000000..1b708db64d8 --- /dev/null +++ b/tests/integration/observability/test_custom_logger_hook_counts_post_call_block.py @@ -0,0 +1,75 @@ +"""A CustomLogger with both hooks gets one failure hook when a post-call guardrail blocks the response. + +The proxy drops the async success logging it parked for the guardrails and logs the block as a failure through both +the async and the sync failure handlers. The logger gets only its async failure hook from them. Speech and +transcription responses never reach post-call guardrails, and image generation logs its success before they run, so +neither parks. + +A legacy sync callback is configured, because that is what makes the sync handler run for an async call.""" + +import uuid +from collections.abc import Iterator +from typing import Final + +import pytest +from integration._support.client import Gateway, gateway_from_environment, held +from integration._support.hook_counts import ( + SURFACE_CALL_TYPES, + HookRecords, + probe_hooks, + provider, + send, + write_hook_count_config, +) +from integration._support.process import owned_proxy +from integration._support.wire import Reply, wire_server + +pytestmark = pytest.mark.timeout(240) + +_PROBES: Final = "integration._support.hook_count_probe" +_PARKED_FOR_POST_CALL_GUARDRAILS: Final = sorted( + SURFACE_CALL_TYPES.keys() - {"speech", "transcription", "image_generation"} +) + + +@pytest.fixture(scope="module") +def post_call_block(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, HookRecords]]: + tmp_path: Final = tmp_path_factory.mktemp("hook-count-post-call-block") + guardrail: Final = { + "guardrail_name": "hook-count-post-call-block", + "litellm_params": { + "guardrail": f"{_PROBES}.BlockingPostCallGuardrail", + "mode": "post_call", + "default_on": True, + }, + } + with ( + wire_server(provider) as upstream, + wire_server(lambda request: Reply()) as sink, + gateway_from_environment() as rig, + ): + config: Final = write_hook_count_config( + tmp_path / "hook-count-post-call-block.yaml", + upstream.url, + {"callbacks": [f"{_PROBES}.probe"], "success_callback": [f"{_PROBES}.sync_success_function"]}, + (guardrail,), + ) + overrides: Final = {"HOOK_COUNT_PROBE_URL": sink.url, "STORE_MODEL_IN_DB": "False"} + with owned_proxy(rig, tmp_path, overrides, config=config, workers=2) as proxy: + yield proxy, HookRecords(sink, function_callbacks=False) + + +@pytest.mark.parametrize("surface", _PARKED_FOR_POST_CALL_GUARDRAILS) +def test_response_a_post_call_guardrail_blocks_gives_the_logger_one_failure_hook( + post_call_block: tuple[Gateway, HookRecords], surface: str +) -> None: + proxy, records = post_call_block + call_id: Final = str(uuid.uuid4()) + response: Final = send(proxy, surface, "success", call_id) + assert response.status_code == 400, response.text + held( + lambda: probe_hooks(records, surface, records.for_call(call_id)), + lambda hooks: hooks == ["async_failure"], + holding=3, + seconds=20, + ) diff --git a/tests/integration/observability/test_custom_logger_hook_counts_vendors.py b/tests/integration/observability/test_custom_logger_hook_counts_vendors.py new file mode 100644 index 00000000000..3c9be9e2a3f --- /dev/null +++ b/tests/integration/observability/test_custom_logger_hook_counts_vendors.py @@ -0,0 +1,128 @@ +"""Built-in loggers that override both hooks deliver one record per async request outside the async flag list: +Datadog one log entry and LangSmith one run. OpenTelemetry keeps its own emit-once guard, so it never exports two +spans for a request either way; its speech, OCR and search spans carry no `litellm.call_id`, so only "never two" +is checked for it. + +A legacy sync callback is configured, because that is what makes the sync handler run for an async call.""" + +import gzip +import json +import uuid +import zlib +from collections.abc import Iterator +from dataclasses import dataclass +from typing import Final + +import pytest +from integration._support.client import Gateway, gateway_from_environment, held +from integration._support.hook_counts import IN_THE_ASYNC_FLAG_LIST, SURFACE_CALL_TYPES, provider, send +from integration._support.hook_counts import write_hook_count_config +from integration._support.otlp_sink import decode_spans +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server + +pytestmark = pytest.mark.timeout(240) + +_PROBES: Final = "integration._support.hook_count_probe" +_FLUSH_SECONDS: Final = 1 +_OUTSIDE_THE_ASYNC_FLAG_LIST: Final = sorted(SURFACE_CALL_TYPES.keys() - IN_THE_ASYNC_FLAG_LIST) + + +@dataclass(frozen=True) +class _Vendors: + proxy: Gateway + sink: Wire + + +@pytest.fixture(scope="module") +def vendors(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Vendors]: + tmp_path: Final = tmp_path_factory.mktemp("custom-logger-hook-count-vendors") + with ( + wire_server(provider) as upstream, + wire_server(lambda request: Reply(status=202)) as sink, + gateway_from_environment() as rig, + ): + config: Final = write_hook_count_config( + tmp_path / "hook-count-vendors.yaml", + upstream.url, + { + "callbacks": ["datadog", "langsmith", "otel"], + "success_callback": [f"{_PROBES}.sync_success_function"], + "failure_callback": [f"{_PROBES}.sync_failure_function"], + }, + ) + overrides: Final = { + "HOOK_COUNT_PROBE_URL": sink.url + "/probe", + "STORE_MODEL_IN_DB": "False", + "DEFAULT_FLUSH_INTERVAL_SECONDS": str(_FLUSH_SECONDS), + "DD_API_KEY": "synthetic-datadog-key", + "DD_SITE": "us5.datadoghq.com", + "DD_BASE_URL": sink.url + "/datadog", + "LANGSMITH_API_KEY": "synthetic-langsmith-key", + "LANGSMITH_PROJECT": "hook-counts", + "LANGSMITH_BASE_URL": sink.url + "/langsmith", + "OTEL_EXPORTER": "otlp_http", + "OTEL_ENDPOINT": sink.url + "/otel/v1/traces", + } + with owned_proxy(rig, tmp_path, overrides, config=config, workers=2) as proxy: + yield _Vendors(proxy, sink) + + +def _unzipped(body: bytes) -> bytes: + if body[:2] == b"\x1f\x8b": + return gzip.decompress(body) + try: + return zlib.decompress(body, zlib.MAX_WBITS | 16) + except zlib.error: + return body + + +def _entries(body: bytes) -> tuple[object, ...]: + decoded: Final = json.loads(_unzipped(body)) + return tuple(decoded) if isinstance(decoded, list) else (decoded,) + + +def _records(request: Request) -> tuple[tuple[str, str], ...]: + """Every record one delivery carries, as (destination, serialized record). A LangSmith batch carries its new + runs under `post`, and an OpenTelemetry export carries every span of the request's trace.""" + if request.target.startswith("/datadog"): + return tuple(("datadog", json.dumps(entry)) for entry in _entries(request.body)) + if request.target.startswith("/langsmith") and request.method == "POST": + runs: Final = tuple( + run + for entry in _entries(request.body) + for run in (entry.get("post", []) if isinstance(entry, dict) and "post" in entry else [entry]) + ) + return tuple(("langsmith", json.dumps(run)) for run in runs) + if request.target.startswith("/otel"): + spans: Final = decode_spans(request.body, request.headers.get("content-type", "")) + return tuple(("otel", json.dumps(span)) for span in spans if "litellm.call_id" in span["attributes"]) + return () + + +@dataclass +class _Deliveries: + sink: Wire + seen: tuple[tuple[str, str], ...] = () + + def per_destination(self, call_id: str) -> dict[str, int]: + self.seen = (*self.seen, *(record for request in self.sink.drain() for record in _records(request))) + destinations: Final = ("datadog", "langsmith", "otel") + return { + destination: sum(1 for name, record in self.seen if name == destination and call_id in record) + for destination in destinations + } + + +@pytest.mark.parametrize("surface", _OUTSIDE_THE_ASYNC_FLAG_LIST) +def test_built_in_loggers_deliver_one_record_per_async_request(vendors: _Vendors, surface: str) -> None: + deliveries: Final = _Deliveries(vendors.sink) + call_id: Final = str(uuid.uuid4()) + response: Final = send(vendors.proxy, surface, "success", call_id) + assert response.status_code == 200, response.text + held( + lambda: deliveries.per_destination(call_id), + lambda counts: counts["datadog"] == 1 and counts["langsmith"] == 1 and counts["otel"] <= 1, + holding=4 * _FLUSH_SECONDS, + seconds=60, + ) diff --git a/tests/integration/routing/test_router_strategies_count_async_requests_once.py b/tests/integration/routing/test_router_strategies_count_async_requests_once.py new file mode 100644 index 00000000000..f7403ad0437 --- /dev/null +++ b/tests/integration/routing/test_router_strategies_count_async_requests_once.py @@ -0,0 +1,214 @@ +"""Router strategies that keep both a sync and an async hook count a finished async request once: least-busy +keeps counting a request still in flight while others finish around it, and usage-based routing v2 adds a +request's tokens to its deployment once. + +A legacy sync callback is configured, because that is what makes the sync handler run for an async call.""" + +import base64 +import json +import os +import threading +import uuid +from collections.abc import Callable, Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, eventually, gateway_from_environment, held +from integration._support.hook_counts import PNG_BYTES, WAV_BYTES, HookRecords +from integration._support.process import OwnedState, owned_proxy +from integration._support.wire import Reply, Request, wire_server +from redis import Redis + +pytestmark = pytest.mark.timeout(240) + +_MODEL: Final = "router-strategy-model" +_PROBES: Final = "integration._support.hook_count_probe" +_REJECTION: Final = b'{"error":{"message":"scripted provider rejection","type":"invalid_request_error"}}' +_IMAGE_TOKENS: Final = 12 +_IMAGE_REPLY: Final = json.dumps( + { + "created": 1700000000, + "data": [{"b64_json": base64.b64encode(PNG_BYTES).decode()}], + "usage": { + "input_tokens": 5, + "output_tokens": 7, + "total_tokens": _IMAGE_TOKENS, + "input_tokens_details": {"text_tokens": 5, "image_tokens": 0}, + }, + } +).encode() + + +@dataclass(frozen=True) +class _Rig: + proxy: Gateway + records: HookRecords + release: threading.Event + deployment_id: str + + +def _provider(release: threading.Event) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + if request.target == "/v1/images/generations": + return Reply(body=_IMAGE_REPLY) + spoken: Final = str(json.loads(request.body or b"{}").get("input", "")) + if spoken.startswith("fail"): + return Reply(status=400, body=_REJECTION) + if spoken.startswith("hold"): + return Reply(chunks=(WAV_BYTES,), content_type="audio/wav", gate_before_first=release) + return Reply(body=WAV_BYTES, content_type="audio/wav") + + return respond + + +def _write_config(path: Path, provider_url: str, strategy: str, model: str, deployment_id: str) -> Path: + path.write_text( + yaml.safe_dump( + { + "model_list": [ + { + "model_name": _MODEL, + "litellm_params": { + "model": model, + "api_base": provider_url + "/v1", + "api_key": "synthetic-provider-key", + }, + "model_info": {"id": deployment_id}, + } + ], + "litellm_settings": { + "success_callback": [f"{_PROBES}.sync_success_function"], + "failure_callback": [f"{_PROBES}.sync_failure_function"], + }, + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + "disable_model_info_refresh": True, + }, + "router_settings": { + "routing_strategy": strategy, + "routing_strategy_args": {"ttl": 600}, + "num_retries": 0, + "disable_cooldowns": True, + "redis_host": "os.environ/REDIS_HOST", + "redis_port": "os.environ/REDIS_PORT", + }, + } + ) + ) + return path + + +@contextmanager +def _router_proxy(tmp_path: Path, strategy: str, model: str) -> Iterator[_Rig]: + release: Final = threading.Event() + deployment_id: Final = f"{strategy}-{uuid.uuid4().hex}" + with ( + wire_server(_provider(release)) as upstream, + wire_server(lambda request: Reply()) as sink, + gateway_from_environment() as rig, + ): + config: Final = _write_config(tmp_path / f"{strategy}.yaml", upstream.url, strategy, model, deployment_id) + try: + with owned_proxy( + rig, + tmp_path, + {"HOOK_COUNT_PROBE_URL": sink.url, "STORE_MODEL_IN_DB": "False"}, + config=config, + workers=2, + state=OwnedState.SHARED_WITH_WORKER, + ) as proxy: + yield _Rig(proxy, HookRecords(sink, function_callbacks=True), release, deployment_id) + finally: + release.set() + + +@pytest.fixture(scope="module") +def least_busy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + with _router_proxy(tmp_path_factory.mktemp("least-busy"), "least-busy", "openai/gpt-4o-mini-tts") as rig: + yield rig + + +@pytest.fixture(scope="module") +def usage_based(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + with _router_proxy(tmp_path_factory.mktemp("usage-based"), "usage-based-routing-v2", "openai/gpt-image-1") as rig: + yield rig + + +def _redis() -> Redis: + return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) + + +def _in_flight(rig: _Rig) -> int: + with _redis() as cache: + stored: Final = cache.get(f"{_MODEL}_request_count:{rig.deployment_id}") + return 0 if stored is None else int(stored) + + +def _tokens_counted(rig: _Rig) -> int: + with _redis() as cache: + keys: Final = tuple(cache.scan_iter(match=f"{rig.deployment_id}:*:tpm:*")) + return sum(int(cache.get(key) or 0) for key in keys) + + +def _post(rig: _Rig, path: str, body: dict[str, object], call_id: str) -> httpx.Response: + return rig.proxy.client.post( + path, + json={"model": _MODEL, **body}, + headers={"Authorization": f"Bearer {rig.proxy.key}", "x-litellm-call-id": call_id}, + timeout=120, + ) + + +def _speak(rig: _Rig, text: str, call_id: str) -> httpx.Response: + return _post(rig, "/v1/audio/speech", {"input": text, "voice": "alloy", "response_format": "wav"}, call_id) + + +def _logged(rig: _Rig, call_id: str) -> None: + """The request's success or failure logging, the legacy sync callback included, has run.""" + eventually(lambda: rig.records.for_call(call_id), bool, seconds=20) + + +def _finish_a_speech_request(rig: _Rig, outcome: str) -> None: + call_id: Final = str(uuid.uuid4()) + finished: Final = _speak(rig, f"{'ok' if outcome == 'success' else 'fail'} {call_id}", call_id) + assert finished.status_code == (200 if outcome == "success" else 400), finished.text + _logged(rig, call_id) + held(lambda: _in_flight(rig), lambda count: count == 1, holding=3, seconds=20) + + +def _generate_an_image(rig: _Rig) -> None: + call_id: Final = str(uuid.uuid4()) + response: Final = _post(rig, "/v1/images/generations", {"prompt": f"tokens {call_id}", "n": 1}, call_id) + assert response.status_code == 200, response.text + _logged(rig, call_id) + + +@pytest.mark.parametrize("outcome", ["success", "failure"]) +def test_least_busy_keeps_counting_a_request_in_flight_while_async_speech_requests_finish( + least_busy: _Rig, outcome: str +) -> None: + held(lambda: _in_flight(least_busy), lambda count: count == 0, holding=1, seconds=30) + least_busy.release.clear() + with ThreadPoolExecutor(max_workers=1) as pool: + in_flight: Final = pool.submit(_speak, least_busy, f"hold {uuid.uuid4().hex}", str(uuid.uuid4())) + eventually(lambda: _in_flight(least_busy), lambda count: count == 1, seconds=30) + try: + for _ in range(3): + _finish_a_speech_request(least_busy, outcome) + finally: + least_busy.release.set() + assert in_flight.result(timeout=60).status_code == 200 + held(lambda: _in_flight(least_busy), lambda count: count == 0, holding=3, seconds=30) + + +def test_usage_based_routing_counts_an_async_image_request_s_tokens_once(usage_based: _Rig) -> None: + for _ in range(3): + _generate_an_image(usage_based) + held(lambda: _tokens_counted(usage_based), lambda tokens: tokens == 3 * _IMAGE_TOKENS, holding=3, seconds=30) diff --git a/tests/test_litellm_rust/messages/test_callbacks.py b/tests/test_litellm_rust/messages/test_callbacks.py index e8d49453065..cd5478dfc3a 100644 --- a/tests/test_litellm_rust/messages/test_callbacks.py +++ b/tests/test_litellm_rust/messages/test_callbacks.py @@ -67,7 +67,7 @@ async def test_success_hook_fires_the_expected_number_of_times( pytest.param("async_log_failure_event", 1), pytest.param("async_post_call_failure_deployment_hook", 1), pytest.param("log_model_group_rate_limit_error", 1), - pytest.param("log_failure_event", 0, marks=SYNC_HANDLERS_ON_ASYNC_CALLS), + pytest.param("log_failure_event", 0), ), ) async def test_failure_hook_fires_the_expected_number_of_times( diff --git a/tests/test_litellm_rust/ocr/test_callbacks.py b/tests/test_litellm_rust/ocr/test_callbacks.py index 5f961288800..18925599acf 100644 --- a/tests/test_litellm_rust/ocr/test_callbacks.py +++ b/tests/test_litellm_rust/ocr/test_callbacks.py @@ -237,7 +237,7 @@ async def test_native_aocr_failure_callbacks_receive_call_type_error_and_no_resp with pytest.raises(litellm.InternalServerError): await call_aocr(server, callbacks=[Observe()]) - assert [observation[0] for observation in observations] == ["sync", "async"] + assert [observation[0] for observation in observations] == ["async"] assert all(observation[1] == "aocr" for observation in observations) assert all(isinstance(observation[2], litellm.InternalServerError) for observation in observations) assert all(observation[3] is None for observation in observations) @@ -278,7 +278,7 @@ async def test_native_aocr_failure_callbacks_receive_state_added_by_pre_call_cal with pytest.raises(litellm.InternalServerError): await call_aocr(server, callbacks=[TrackInFlightRequest()]) - assert [event for event, _ in observed] == ["sync", "async"] + assert [event for event, _ in observed] == ["async"] assert all(observed_token is token for _, observed_token in observed) @@ -445,7 +445,7 @@ async def test_native_aocr_state_stashed_before_a_blocking_hook_raises_reaches_f await call_aocr(server) await drain_logging() - assert observed == [("sync", token, raised.value), ("async", token, raised.value)] + assert observed == [("async", token, raised.value)] @pytest.mark.asyncio diff --git a/tests/test_litellm_rust/ocr/test_lifecycle.py b/tests/test_litellm_rust/ocr/test_lifecycle.py index c35f6836144..611316fb52c 100644 --- a/tests/test_litellm_rust/ocr/test_lifecycle.py +++ b/tests/test_litellm_rust/ocr/test_lifecycle.py @@ -144,7 +144,7 @@ async def test_mapped_failure_identity_and_deployment_snapshot(server: Recording with pytest.raises(litellm.InternalServerError) as caught: await call_aocr(server, callbacks=[recorder]) failures: Final = tuple(event for event in recorder.events if "failure" in event.name) - assert [event.name for event in failures] == ["log_failure_event", "async_log_failure_event"] + assert [event.name for event in failures] == ["async_log_failure_event"] assert all(event.kwargs["exception"] is caught.value for event in failures) assert caught.value.status_code == 500 assert snapshots[0] is not caught.value @@ -324,24 +324,26 @@ async def test_failure_callbacks_continue_within_both_families(server: Recording server.enqueue(ResponseSpec(body={"message": "failed"}, status=500)) observed: Final = [] - class Broken(CustomLogger): + class BrokenSync(CustomLogger): def log_failure_event(self, kwargs, response_obj, start_time, end_time): observed.append(("broken-sync", kwargs["exception"])) raise RuntimeError("sync observer") + class FollowingSync(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): + observed.append(("following-sync", kwargs["exception"])) + + class BrokenAsync(CustomLogger): async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): observed.append(("broken-async", kwargs["exception"])) raise RuntimeError("async observer") - class Following(CustomLogger): - def log_failure_event(self, kwargs, response_obj, start_time, end_time): - observed.append(("following-sync", kwargs["exception"])) - + class FollowingAsync(CustomLogger): async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): observed.append(("following-async", kwargs["exception"])) with pytest.raises(litellm.InternalServerError) as caught: - await call_aocr(server, callbacks=[Broken(), Following()]) + await call_aocr(server, callbacks=[BrokenSync(), FollowingSync(), BrokenAsync(), FollowingAsync()]) assert [name for name, _ in observed] == ["broken-sync", "following-sync", "broken-async", "following-async"] assert all(error is caught.value for _, error in observed) diff --git a/tests/test_litellm_rust/responses/test_callbacks.py b/tests/test_litellm_rust/responses/test_callbacks.py index a4abdd43529..0e921e9d5db 100644 --- a/tests/test_litellm_rust/responses/test_callbacks.py +++ b/tests/test_litellm_rust/responses/test_callbacks.py @@ -95,7 +95,7 @@ async def test_success_hook_fires_the_expected_number_of_times( "async_post_call_failure_deployment_hook", True, 1, id="async-async_post_call_failure_deployment_hook" ), pytest.param("log_model_group_rate_limit_error", True, 1, id="async-log_model_group_rate_limit_error"), - pytest.param("log_failure_event", True, 0, id="async-log_failure_event", marks=SYNC_HANDLERS_ON_ASYNC_CALLS), + pytest.param("log_failure_event", True, 0, id="async-log_failure_event"), pytest.param("log_failure_event", False, 1, id="sync-log_failure_event"), ), ) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 981c43a9792..b3414ebe9d7 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -3088,7 +3088,7 @@ async def test_dispatch_failure_handlers_submits_sync_handler_for_failure_only_c prefer_async_handlers=True, ) - mock_submit.assert_called_once_with(mock_sync, exception, traceback_exception) + mock_submit.assert_called_once_with(mock_sync, exception, traceback_exception, for_async_request=True) @pytest.mark.asyncio @@ -8188,6 +8188,155 @@ def test_cache_callback_alone_does_not_start_sync_callbacks_for_async_requests( assert logging_obj._should_run_sync_callbacks_for_async_calls() is expected +class _FailureHookRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.hooks: list[str] = [] + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.hooks.append("sync_failure") + + +class _BothFailureHooksRecorder(_FailureHookRecorder): + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.hooks.append("async_failure") + + +def _failure_hook_count_logging_obj( + call_type: str, recorder: _FailureHookRecorder, *, async_failure_listed: bool +) -> LitellmLogging: + logging_obj: Final = LitellmLogging( + model="gpt-4o-mini-tts", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type=call_type, + start_time=datetime.datetime.now(), + litellm_call_id=f"failure-hook-count-{call_type}", + function_id=f"failure-hook-count-{call_type}", + dynamic_failure_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder] if async_failure_listed else [], + ) + logging_obj.update_environment_variables(litellm_params={"metadata": {}}, optional_params={}) + return logging_obj + + +@pytest.fixture +def no_global_callbacks(monkeypatch: pytest.MonkeyPatch) -> None: + for name in ( + "callbacks", + "success_callback", + "failure_callback", + "_async_success_callback", + "_async_failure_callback", + ): + monkeypatch.setattr(litellm, name, []) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("recorder_type", "async_listed", "expected"), + [ + (_BothFailureHooksRecorder, True, ["async_failure"]), + (_FailureHookRecorder, True, ["sync_failure"]), + (_BothFailureHooksRecorder, False, ["sync_failure"]), + ], +) +@pytest.mark.parametrize("call_type", ["aspeech", "atranscription", "aimage_edit", "asearch"]) +@pytest.mark.usefixtures("no_global_callbacks") +async def test_async_call_outside_the_async_flag_list_gets_one_custom_logger_failure_hook( + call_type: str, recorder_type: type[_FailureHookRecorder], async_listed: bool, expected: list[str] +) -> None: + recorder: Final = recorder_type() + logging_obj: Final = _failure_hook_count_logging_obj(call_type, recorder, async_failure_listed=async_listed) + + logging_obj.failure_handler(ValueError("upstream rejected"), "traceback", for_async_request=True) + await logging_obj.async_failure_handler(ValueError("upstream rejected"), "traceback") + + assert recorder.hooks == expected + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_global_callbacks") +async def test_logger_in_the_global_failure_callback_lists_gets_one_failure_hook( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder: Final = _BothFailureHooksRecorder() + monkeypatch.setattr(litellm, "failure_callback", [recorder]) + monkeypatch.setattr(litellm, "_async_failure_callback", [recorder]) + logging_obj: Final = LitellmLogging( + model="gpt-4o-mini-tts", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="aspeech", + start_time=datetime.datetime.now(), + litellm_call_id="global-failure", + function_id="global-failure", + ) + logging_obj.update_environment_variables(litellm_params={"metadata": {}}, optional_params={}) + + logging_obj.failure_handler(ValueError("upstream rejected"), "traceback", for_async_request=True) + await logging_obj.async_failure_handler(ValueError("upstream rejected"), "traceback") + + assert recorder.hooks == ["async_failure"] + + +@pytest.mark.usefixtures("no_global_callbacks") +def test_failure_handler_run_without_an_async_failure_run_keeps_the_sync_hook() -> None: + recorder: Final = _BothFailureHooksRecorder() + + _failure_hook_count_logging_obj("aspeech", recorder, async_failure_listed=True).failure_handler( + ValueError("upstream rejected"), "traceback" + ) + + assert recorder.hooks == ["sync_failure"] + + +@pytest.mark.usefixtures("no_global_callbacks") +def test_async_call_outside_the_async_flag_list_still_runs_sync_failure_function_callbacks( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm_logging, "customLogger", CustomLogger()) + calls: Final[list[str]] = [] + + def failure_function(kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + calls.append("function_failure") + + recorder: Final = _BothFailureHooksRecorder() + logging_obj: Final = _failure_hook_count_logging_obj("aspeech", recorder, async_failure_listed=True) + logging_obj.dynamic_failure_callbacks = [recorder, failure_function] + + logging_obj.failure_handler(ValueError("upstream rejected"), "traceback", for_async_request=True) + + assert calls == ["function_failure"] + assert recorder.hooks == [] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("no_global_callbacks") +async def test_client_wrapped_failure_gives_a_logger_in_litellm_callbacks_one_hook( + monkeypatch: pytest.MonkeyPatch, +) -> None: + recorder: Final = _BothFailureHooksRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + + @litellm.utils.client + async def aspeech(model: str, input: str, voice: str, **kwargs: object) -> None: + raise litellm.BadRequestError(message="upstream rejected", model=model, llm_provider="openai") + + @litellm.utils.client + def speech(model: str, input: str, voice: str, **kwargs: object) -> None: + raise litellm.BadRequestError(message="upstream rejected", model=model, llm_provider="openai") + + with pytest.raises(litellm.BadRequestError): + await aspeech(model="gpt-4o-mini-tts", input="hi", voice="alloy") + with pytest.raises(litellm.BadRequestError): + speech(model="gpt-4o-mini-tts", input="hi", voice="alloy") + + assert recorder.hooks == ["async_failure", "sync_failure"] + + @pytest.mark.asyncio @pytest.mark.parametrize("turn_off_message_logging", [False, True]) @pytest.mark.parametrize("make_response", [ModelResponse, _responses_api_response_with_text]) diff --git a/tests/unit/proxy/test_proxy_reject_logging.py b/tests/unit/proxy/test_proxy_reject_logging.py index 6e8545a58b3..b300d48808f 100644 --- a/tests/unit/proxy/test_proxy_reject_logging.py +++ b/tests/unit/proxy/test_proxy_reject_logging.py @@ -209,4 +209,4 @@ async def test_chat_completion_request_with_redaction(route, body, router, monke assert _test_logger.reaches_async_failure_event is True - assert _test_logger.reaches_sync_failure_event is True + assert _test_logger.reaches_sync_failure_event is False diff --git a/tests/unit/proxy/test_proxy_utils.py b/tests/unit/proxy/test_proxy_utils.py index 2fc1b72c932..02f31bf07c6 100644 --- a/tests/unit/proxy/test_proxy_utils.py +++ b/tests/unit/proxy/test_proxy_utils.py @@ -2652,6 +2652,58 @@ async def test_post_call_failure_hook_auth_error_llm_api_route(): mock_handle_logging.assert_called_once() +class _InlineThread: + def __init__(self, target, args=(), kwargs=None, daemon=None) -> None: + self._run = lambda: target(*args, **(kwargs or {})) + + def start(self) -> None: + self._run() + + +@pytest.mark.asyncio +async def test_proxy_only_failure_gives_a_logger_with_both_failure_hooks_only_the_async_one( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import types + + import litellm.proxy.utils as proxy_utils + from litellm.integrations.custom_logger import CustomLogger + from litellm.litellm_core_utils.litellm_logging import Logging + + hooks: Final[list[str]] = [] + + class Recorder(CustomLogger): + def log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + hooks.append("sync_failure") + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + hooks.append("async_failure") + + for name in ("failure_callback", "_async_failure_callback"): + monkeypatch.setattr(litellm, name, []) + monkeypatch.setattr(proxy_utils, "threading", types.SimpleNamespace(Thread=_InlineThread)) + recorder: Final = Recorder() + logging_obj: Final = Logging( + model="gpt-4o-mini-tts", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="aspeech", + start_time=datetime.now(), + litellm_call_id="proxy-only-failure", + function_id="proxy-only-failure", + dynamic_failure_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables(litellm_params={"metadata": {}}, optional_params={}) + + await proxy_utils.ProxyLogging._dispatch_proxy_only_failure_handlers( + litellm_logging_obj=logging_obj, + original_exception=HTTPException(status_code=401, detail="Authentication Error: invalid user key"), + ) + + assert hooks == ["async_failure"] + + @pytest.mark.asyncio @pytest.mark.parametrize( "request_data, route, expected_call_type", diff --git a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py index eb1bfd82715..a40f8a77105 100644 --- a/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/host/test_callbacks_legacy_python.py @@ -9,6 +9,7 @@ import pytest from pydantic import TypeAdapter from litellm._internal_context import is_internal_call +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging from litellm.rust_bridge.host import callbacks_legacy_python as legacy from litellm.rust_bridge.host.callbacks_legacy_python import failure_handler, setup @@ -167,6 +168,75 @@ def test_failure_handler_of_an_internal_call_leaves_the_outer_budget_reservation pending.close() +class _Hooks(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.hooks: list[str] = [] + + def log_success_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.hooks.append("sync_success") + + async def async_log_success_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.hooks.append("async_success") + + def log_failure_event(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.hooks.append("sync_failure") + + async def async_log_failure_event( + self, kwargs: object, response_obj: object, start_time: object, end_time: object + ) -> None: + self.hooks.append("async_failure") + + +def _logger_with_hooks(hooks: _Hooks) -> Logging: + logger: Final = Logging( + model="mistral/mistral-ocr-latest", + messages=[], + stream=False, + call_type="aocr", + start_time=datetime.datetime.now(), + litellm_call_id="hooks", + function_id="hooks", + dynamic_success_callbacks=[hooks], + dynamic_async_success_callbacks=[hooks], + dynamic_failure_callbacks=[hooks], + dynamic_async_failure_callbacks=[hooks], + ) + logger.update_environment_variables(litellm_params={"metadata": {}}, optional_params={}) + return logger + + +@pytest.mark.asyncio +async def test_failure_of_an_async_call_gives_a_logger_with_both_failure_hooks_only_the_async_one() -> None: + hooks: Final = _Hooks() + now: Final = datetime.datetime.now() + kwargs: Final = {**_OCR_KWARGS, "litellm_logging_obj": _logger_with_hooks(hooks)} + result: Final = setup("aocr", (), kwargs, now, asynchronous=True) + + assert failure_handler(result.logger, RuntimeError("upstream refused"), now, now, asynchronous=False) is None + pending: Final = failure_handler(result.logger, RuntimeError("upstream refused"), now, now, asynchronous=True) + assert pending is not None + await pending + + assert hooks.hooks == ["async_failure"] + + +@pytest.mark.parametrize("earlier_async_call", [False, True]) +def test_failure_of_a_sync_call_keeps_the_sync_failure_hook(earlier_async_call: bool) -> None: + hooks: Final = _Hooks() + now: Final = datetime.datetime.now() + kwargs: Final = {**_OCR_KWARGS, "litellm_logging_obj": _logger_with_hooks(hooks)} + if earlier_async_call: + setup("aocr", (), kwargs, now, asynchronous=True) + result: Final = setup("ocr", (), kwargs, now, asynchronous=False) + + assert failure_handler(result.logger, RuntimeError("upstream refused"), now, now, asynchronous=False) is None + + assert hooks.hooks == ["sync_failure"] + + CONTRACT_PATH: Final = Path(__file__).parents[4] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" From 0c6731eb5c2cc2c07abddb177cffef3a5b273dfe Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 15:20:16 -0700 Subject: [PATCH 010/235] fix(router): price cost-based routing through model_info.base_model (#47617) * fix(router): price cost-based routing through model_info.base_model * fix(router): keep cost-based routing pricing helpers fully typed * fix(router): drop typing.cast from cost-based routing pricing helpers * fix(router): read the cost map through a typed accessor in cost-based routing * fix(router): avoid a Final rebind inside the cost-based routing loop * refactor(router): share deployment_pricing_models across spend, listing and cost-based routing * refactor(router): keep the /v1/models listing key inline, share pricing models for spend and routing * refactor(router): inline the deployment pricing-model wrapper in cost-based routing * refactor(router): read deployment pricing models through one shared helper deployment_pricing_models(deployment) now owns the model_info.base_model / litellm_params.base_model / litellm_params.model unpacking, used by cost-based routing and the /v1/models listing. pricing_models(base_model, model) stays for the failure-row spend estimate, which already holds the two names * refactor(router): share the str-keyed mapping validator with cost-based routing * ci: re-run after flaky test_aiohttp_transport read timeout in llms-providers --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 5 +- litellm/router.py | 17 ++---- litellm/router_strategy/lowest_cost.py | 33 ++++++++++-- litellm/router_utils/deployment_pricing.py | 37 +++++++++++++ .../unit/router_strategy/test_lowest_cost.py | 46 ++++++++++++++++ .../router_utils/test_deployment_pricing.py | 53 +++++++++++++++++++ 6 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 litellm/router_utils/deployment_pricing.py create mode 100644 tests/unit/router_utils/test_deployment_pricing.py diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c5983077d4c..c51e76123e4 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -269,6 +269,7 @@ from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) from litellm.router_utils.common_utils import resolve_model_group_alias +from litellm.router_utils.deployment_pricing import pricing_models from litellm.secret_managers.main import str_to_bool from litellm.types.integrations.slack_alerting import DEFAULT_ALERT_TYPES from litellm.types.llms.openai import ResponsesAPIResponse @@ -1245,9 +1246,9 @@ def _estimated_input_cost( deployment_pricing: Final = deployment_pricing_model_info(model_id=deployment_id, deployment_model=model) if deployment_pricing is not None: return _deployment_priced_input_cost(model, custom_llm_provider, prompt_tokens, deployment_pricing) - pricing_models: Final = (base_model, model) if base_model is not None else (model,) + pricing_models_to_try: Final = pricing_models(base_model, model) candidate_costs: Final = ( - _priced_input_cost(pricing_model, custom_llm_provider, prompt_tokens) for pricing_model in pricing_models + _priced_input_cost(pricing_model, custom_llm_provider, prompt_tokens) for pricing_model in pricing_models_to_try ) return next((cost for cost in candidate_costs if cost is not None), 0.0) diff --git a/litellm/router.py b/litellm/router.py index 74facf200d2..1c5a9176207 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -242,6 +242,7 @@ from litellm.router_utils.cooldown_handlers import ( is_video_lookup_not_found, set_cooldown_deployments, ) +from litellm.router_utils.deployment_pricing import deployment_pricing_models from litellm.router_utils.embedding_batch_size_check import ( async_log_rejection_before_provider_call, clear_rejected_before_provider_call, @@ -11687,19 +11688,9 @@ class Router: ) for deployment in deployments ) - params: Final = tuple(deployment.get("litellm_params") or MappingProxyType({}) for deployment in deployments) - # base_model resolution mirrors get_router_model_info: unset or blank means the - # deployment's own model name is the cost-map key. - cost_map_keys: Final = tuple( - dict.fromkeys( # deduplicates while preserving config order - key - for key in ( - model_info.get("base_model") or litellm_params.get("base_model") or litellm_params.get("model") - for model_info, litellm_params in zip(model_infos, params) - ) - if isinstance(key, str) and key - ) - ) + raw_deployments: Final[tuple[object, ...]] = deployments + pricing_keys: Final = (deployment_pricing_models(deployment) for deployment in raw_deployments) + cost_map_keys: Final = tuple(dict.fromkeys(keys[0] for keys in pricing_keys if keys)) return DeploymentModelListingInfo( cost_map_keys=cost_map_keys, max_input_tokens=self._widest_configured_limit(model_infos, "max_input_tokens"), diff --git a/litellm/router_strategy/lowest_cost.py b/litellm/router_strategy/lowest_cost.py index d567b6acccc..5894da9cb4f 100644 --- a/litellm/router_strategy/lowest_cost.py +++ b/litellm/router_strategy/lowest_cost.py @@ -1,5 +1,6 @@ #### What this does #### # picks based on response time (for streaming, this is time to first token) +from collections.abc import Mapping from datetime import datetime from typing import Final @@ -10,6 +11,30 @@ from litellm._logging import verbose_router_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.router_utils.batch_utils import is_batch_retrieve_call_type +from litellm.router_utils.deployment_pricing import deployment_pricing_models, str_keyed + + +def _deployment_cost_map_rates(deployment: object) -> tuple[float | None, float | None]: + """Input and output per-token prices from the first cost-map row among the deployment's + pricing models that carries one; (None, None) when no row does, so the caller falls back to + its unmapped default.""" + rows: Final = (str_keyed(_cost_map().get(model)) for model in deployment_pricing_models(deployment)) + row: Final = next((row for row in rows if _declares_token_price(row)), None) + if row is None: + return None, None + return _as_rate(row.get("input_cost_per_token")), _as_rate(row.get("output_cost_per_token")) + + +def _cost_map() -> Mapping[str, object]: + return litellm.model_cost + + +def _declares_token_price(row: Mapping[str, object]) -> bool: + return row.get("input_cost_per_token") is not None or row.get("output_cost_per_token") is not None + + +def _as_rate(value: object) -> float | None: + return float(value) if isinstance(value, (int, float)) and not isinstance(value, bool) else None class LowestCostLoggingHandler(CustomLogger): @@ -243,8 +268,8 @@ class LowestCostLoggingHandler(CustomLogger): or _deployment.get("model_info", {}).get("rpm", None) or float("inf") ) - item_litellm_model_name = _deployment.get("litellm_params", {}).get("model") - item_litellm_model_cost_map = litellm.model_cost.get(item_litellm_model_name, {}) + deployment_view: object = _deployment + map_input_cost, map_output_cost = _deployment_cost_map_rates(deployment_view) # check if user provided input_cost_per_token and output_cost_per_token in litellm_params item_input_cost = None @@ -256,10 +281,10 @@ class LowestCostLoggingHandler(CustomLogger): item_output_cost = _deployment.get("litellm_params", {}).get("output_cost_per_token") if item_input_cost is None: - item_input_cost = item_litellm_model_cost_map.get("input_cost_per_token", 5.0) + item_input_cost = map_input_cost if map_input_cost is not None else 5.0 if item_output_cost is None: - item_output_cost = item_litellm_model_cost_map.get("output_cost_per_token", 5.0) + item_output_cost = map_output_cost if map_output_cost is not None else 5.0 # if litellm["model"] is not in model_cost map -> use item_cost = $10 diff --git a/litellm/router_utils/deployment_pricing.py b/litellm/router_utils/deployment_pricing.py new file mode 100644 index 00000000000..622cc91ab5e --- /dev/null +++ b/litellm/router_utils/deployment_pricing.py @@ -0,0 +1,37 @@ +from collections.abc import Iterator, Mapping +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STR_KEYED: Final = TypeAdapter(dict[str, object]) + + +def pricing_models(base_model: object, model: object) -> tuple[str, ...]: + """Cost-map keys to price a deployment under, ``model_info.base_model`` first, then the + deployment's own model, blanks and duplicates dropped. Every spend and routing surface that + prices a deployment walks this order, so a ``base_model`` the cost map does not know falls + through to the deployment model instead of billing or ranking at the unmapped default.""" + return tuple(dict.fromkeys(_named_models(base_model, model))) + + +def deployment_pricing_models(deployment: object) -> tuple[str, ...]: + """``pricing_models`` read off a raw deployment dict (``model_info.base_model``, then + ``litellm_params.base_model``, then ``litellm_params.model``); malformed blocks count as unset.""" + view: Final = str_keyed(deployment) + params: Final = str_keyed(view.get("litellm_params")) + info: Final = str_keyed(view.get("model_info")) + return pricing_models(info.get("base_model") or params.get("base_model"), params.get("model")) + + +def _named_models(*candidates: object) -> Iterator[str]: + return (candidate for candidate in candidates if isinstance(candidate, str) and candidate) + + +def str_keyed(value: object) -> Mapping[str, object]: + """``value`` as a read-only str-keyed mapping, empty when it is not a dict.""" + try: + return MappingProxyType(_STR_KEYED.validate_python(value)) + except ValidationError: + return _EMPTY diff --git a/tests/unit/router_strategy/test_lowest_cost.py b/tests/unit/router_strategy/test_lowest_cost.py index b53ec6e6ba2..e3fdcd69499 100644 --- a/tests/unit/router_strategy/test_lowest_cost.py +++ b/tests/unit/router_strategy/test_lowest_cost.py @@ -112,6 +112,7 @@ def _vcr_outcome_gate(request, vcr): yield record_vcr_outcome(request, vcr) + @pytest.fixture(scope="function") def isolate_litellm_state(): """ @@ -164,6 +165,7 @@ def isolate_litellm_state(): setattr(litellm, attr, original_value) _invalidate_model_cost_lowercase_map() + _SCALAR_DEFAULTS = { "num_retries": getattr(litellm, "num_retries", None), "num_retries_per_request": getattr(litellm, "num_retries_per_request", None), @@ -184,6 +186,7 @@ _SCALAR_DEFAULTS = { "api_key": getattr(litellm, "api_key", None), } + @pytest.fixture(scope="module") def setup_and_teardown(): """ @@ -206,6 +209,7 @@ def setup_and_teardown(): litellm.in_memory_llm_clients_cache.flush_cache() yield + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_get_available_deployments(): @@ -235,6 +239,7 @@ async def test_get_available_deployments(): assert selected_model["model_info"]["id"] == "groq-llama" + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.asyncio async def test_get_available_deployments_custom_price(): @@ -286,6 +291,7 @@ async def test_get_available_deployments_custom_price(): assert selected_model["model_info"]["id"] == "chatgpt-v-1" + async def _deploy(lowest_cost_logger, deployment_id, tokens_used, duration): kwargs = { "litellm_params": { @@ -307,6 +313,7 @@ async def _deploy(lowest_cost_logger, deployment_id, tokens_used, duration): end_time=end_time, ) + @pytest.mark.usefixtures("_vcr_outcome_gate", "isolate_litellm_state", "setup_and_teardown") @pytest.mark.parametrize("ans_rpm", [1, 5]) # 1 should produce nothing, 10 should select first @pytest.mark.asyncio @@ -357,3 +364,42 @@ async def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm): assert (d_ans and d_ans["model_info"]["id"]) == ans print("selected deployment:", d_ans) + + +_TWO_DOLLARS_PER_TOKEN = {"input_cost_per_token": 1.0, "output_cost_per_token": 1.0} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "azure_litellm_params, base_model", + [ + ({"model": "azure/my-gpt4-deploy"}, "azure/gpt-4.1"), + ({"model": "azure/gpt-4.1"}, "azure/not-a-real-model"), + ], + ids=["base_model_priced", "unmapped_base_model_falls_back_to_deployment_model"], +) +async def test_cost_based_routing_prices_an_azure_deployment_through_its_base_model( + azure_litellm_params: dict, base_model: str +): + handler = LowestCostLoggingHandler(router_cache=DualCache()) + healthy_deployments = [ + { + "model_name": "gpt-4.1", + "litellm_params": {"model": "openai/gpt-4.1", **_TWO_DOLLARS_PER_TOKEN}, + "model_info": {"id": "openai-two-dollars"}, + }, + { + "model_name": "gpt-4.1", + "litellm_params": azure_litellm_params, + "model_info": {"id": "azure-cost-map-priced", "base_model": base_model}, + }, + ] + + picked = await handler.async_get_available_deployments( + model_group="gpt-4.1", healthy_deployments=healthy_deployments + ) + + assert picked is not None and picked["model_info"]["id"] == "azure-cost-map-priced", ( + "a cost-map priced Azure deployment must beat a $2 per token deployment, " + "it loses only when it is scored at the $10 unmapped default" + ) diff --git a/tests/unit/router_utils/test_deployment_pricing.py b/tests/unit/router_utils/test_deployment_pricing.py new file mode 100644 index 00000000000..45cc211a89c --- /dev/null +++ b/tests/unit/router_utils/test_deployment_pricing.py @@ -0,0 +1,53 @@ +import pytest + +from litellm.router_utils.deployment_pricing import deployment_pricing_models, pricing_models + + +@pytest.mark.parametrize( + "base_model, model, expected", + [ + ("azure/gpt-4.1", "azure/my-gpt4-deploy", ("azure/gpt-4.1", "azure/my-gpt4-deploy")), + (None, "azure/my-gpt4-deploy", ("azure/my-gpt4-deploy",)), + ("", "azure/my-gpt4-deploy", ("azure/my-gpt4-deploy",)), + ("azure/gpt-4.1", "azure/gpt-4.1", ("azure/gpt-4.1",)), + (None, None, ()), + ], + ids=["base_model_first", "no_base_model", "blank_base_model", "same_name_once", "nothing_named"], +) +def test_pricing_models_walks_base_model_then_deployment_model(base_model, model, expected): + assert pricing_models(base_model, model) == expected + + +@pytest.mark.parametrize( + "deployment, expected", + [ + ( + {"litellm_params": {"model": "azure/my-gpt4-deploy"}, "model_info": {"base_model": "azure/gpt-4.1"}}, + ("azure/gpt-4.1", "azure/my-gpt4-deploy"), + ), + ( + {"litellm_params": {"model": "azure/my-gpt4-deploy", "base_model": "azure/gpt-4.1"}}, + ("azure/gpt-4.1", "azure/my-gpt4-deploy"), + ), + ( + { + "litellm_params": {"model": "azure/my-gpt4-deploy", "base_model": "azure/gpt-4o"}, + "model_info": {"base_model": "azure/gpt-4.1"}, + }, + ("azure/gpt-4.1", "azure/my-gpt4-deploy"), + ), + ({"litellm_params": {"model": "openai/gpt-4.1"}, "model_info": {}}, ("openai/gpt-4.1",)), + ({"litellm_params": "not-a-dict", "model_info": None}, ()), + ("not-a-deployment", ()), + ], + ids=[ + "model_info_base_model", + "params_base_model", + "model_info_wins", + "no_base_model", + "malformed_blocks", + "not_a_dict", + ], +) +def test_deployment_pricing_models_reads_the_deployment_dict(deployment, expected): + assert deployment_pricing_models(deployment) == expected From 27efc387f7cc6bf36cacbf6eb88e39e488d240e7 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 15:26:09 -0700 Subject: [PATCH 011/235] fix(proxy): price earlier timed-out retry attempts at their own deployment's model rate (#47648) * fix(proxy): bill every earlier timed-out retry attempt at its own deployment's model rate * fix(router): read the retry record's base_model through the typed metadata helper * fix(router): keep the four-record retry breadcrumb cap * fix(router): carry a deployment's litellm_params base_model on its retry record * chore(router): place the cast-ok reason on the cast line and list the base_model helper as covered through log_retry --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/utils.py | 56 +++++++++++---- litellm/router.py | 21 +++++- litellm/types/router.py | 2 + .../router_code_coverage.py | 1 + ..._utils_model_creation_and_error_logging.py | 68 +++++++++++++++++++ tests/unit/test_router/test_router.py | 2 +- .../test_router/test_router_helper_utils.py | 41 ++++++++++- 7 files changed, 174 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c51e76123e4..b5101898bb1 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1161,6 +1161,43 @@ def _timed_out_earlier_records( return tuple(record for identity, record in by_identity.items() if identity != final_identity) +def _record_pricing( + model_call_details: Mapping[str, object], record: Mapping[str, object], base_model: str | None +) -> tuple[Mapping[str, object], str | None]: + """The ``(model_call_details, base_model)`` to price one earlier attempt with: the record's own + deployment model and ``base_model`` when it served a different model than the final attempt + (the provider is then inferred from that model), else the final attempt's.""" + record_model: Final = record.get("model") + if not isinstance(record_model, str) or record_model == model_call_details.get("model"): + return model_call_details, base_model + record_base_model: Final = record.get("base_model") + return ( + {**model_call_details, "model": record_model, "custom_llm_provider": None}, + record_base_model if isinstance(record_base_model, str) else None, + ) + + +def _billed_earlier_attempt( + model_call_details: Mapping[str, object], + record: Mapping[str, object], + index: int, + estimated_usage: Usage, + base_model: str | None, +) -> BilledFailedAttempt: + pricing_details, pricing_base_model = _record_pricing(model_call_details, record, base_model) + return BilledFailedAttempt( + attempted_retries=_attempt_number(record, index), + deployment_id=_record_deployment_id(record), + usage=estimated_usage, + response_cost=_estimated_input_cost( + pricing_details, + estimated_usage.prompt_tokens, + deployment_id=_record_deployment_id(record), + base_model=pricing_base_model, + ), + ) + + def _billed_earlier_attempts( model_call_details: Mapping[str, object], litellm_params: Mapping[str, object] | None, @@ -1174,23 +1211,14 @@ def _billed_earlier_attempts( request's own deployment and retry count (``attempted_retries`` restarts at 0 after a fallback hop, so the count alone cannot tell the groups apart). An earlier attempt that ended in a ``Timeout`` had the same prompt accepted by its deployment, so it is billed like the final - attempt, at that deployment's input rate (else the cost map's rate for ``base_model`` or the - model). Only the exception class survives on the record, so an earlier connect timeout is - billed too; any other failure class is not.""" + attempt, at that deployment's input rate (else the cost map's rate for the record's own + ``base_model`` or model, which a fallback hop can make a different model than the final one). + Only the exception class survives on the record, so an earlier connect timeout is billed too; + any other failure class is not.""" final_attempt: Final = _retry_metadata_bucket(litellm_params).get("attempted_retries") final_identity: Final = (deployment_id, final_attempt if isinstance(final_attempt, int) else None) return tuple( - BilledFailedAttempt( - attempted_retries=_attempt_number(record, index), - deployment_id=_record_deployment_id(record), - usage=estimated_usage, - response_cost=_estimated_input_cost( - model_call_details, - estimated_usage.prompt_tokens, - deployment_id=_record_deployment_id(record), - base_model=base_model, - ), - ) + _billed_earlier_attempt(model_call_details, record, index, estimated_usage, base_model) for index, record in enumerate(_timed_out_earlier_records(litellm_params, final_identity)) ) diff --git a/litellm/router.py b/litellm/router.py index 1c5a9176207..679f4eba1ea 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9498,17 +9498,24 @@ class Router: When a retry or fallback happens, record which model group, deployment and attempt just failed and why, and count it toward the request-wide num_retries_per_request cap """ + from litellm.litellm_core_utils.get_litellm_params import get_base_model_from_litellm_call_metadata from litellm.types.router import RetryAttemptRecord _metadata_var: Final = "litellm_metadata" if "litellm_metadata" in kwargs else "metadata" - request_metadata: Final[Mapping[str, object]] = kwargs[_metadata_var] + request_metadata: Final = cast("Mapping[str, object]", kwargs[_metadata_var]) # cast-ok: untyped kwargs model_group: Final = kwargs.get("model") model_info: Final = request_metadata.get("model_info") deployment_id: Final = model_info.get("id") if isinstance(model_info, Mapping) else None + base_model: Final = get_base_model_from_litellm_call_metadata( + metadata=request_metadata + ) or self._deployment_params_base_model(deployment_id if isinstance(deployment_id, str) else None) + deployment_model: Final = request_metadata.get("deployment") attempted_retries: Final = request_metadata.get("attempted_retries") attempt_record: Final[RetryAttemptRecord] = { "model_group": model_group if isinstance(model_group, str) else None, "deployment_id": deployment_id if isinstance(deployment_id, str) else None, + "model": deployment_model if isinstance(deployment_model, str) else None, + "base_model": base_model if isinstance(base_model, str) else None, "exception_type": type(e).__name__, "exception_string": str(e), "attempted_retries": attempted_retries if type(attempted_retries) is int else None, @@ -9539,6 +9546,18 @@ class Router: kwargs[_metadata_var]["request_retry_count"] = request_retry_count # rebind-ok: same dict, read by the cap return kwargs + def _deployment_params_base_model(self, deployment_id: str | None) -> str | None: + """The ``base_model`` a deployment declares under ``litellm_params`` (the Azure named-deployment + shape), the same key the success path's ``get_base_model_from_metadata`` reads.""" + deployment: Final = self.get_deployment(model_id=deployment_id) if deployment_id is not None else None + if deployment is None: + return None + extra_params: Final = cast( # cast-ok: pydantic extras are untyped, base_model is isinstance-checked below + "Mapping[str, object]", deployment.litellm_params.model_extra or {} + ) + base_model: Final = extra_params.get("base_model") + return base_model if isinstance(base_model, str) else None + def _update_usage(self, deployment_id: str, parent_otel_span: Span | None) -> int: """ Update deployment rpm for that minute diff --git a/litellm/types/router.py b/litellm/types/router.py index 97b0d81d33b..cd1389aea9d 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1184,6 +1184,8 @@ class RouterModelGroupAliasItem(TypedDict): class RetryAttemptRecord(TypedDict): model_group: ReadOnly[str | None] deployment_id: ReadOnly[str | None] + model: ReadOnly[str | None] + base_model: ReadOnly[str | None] exception_type: ReadOnly[str] exception_string: ReadOnly[str] attempted_retries: ReadOnly[int | None] diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 7809a23e955..55435c51648 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -93,6 +93,7 @@ ignored_function_names = [ "_apply_updated_routing_strategy_args", # Tested via update_settings in test_lowest_latency.py (file lacks "router" in name) "_wait_for_scheduler_turn", # Tested through prioritized acompletion and atext_completion in test_router.py "arm_routing_read_prefetch", # Tested in tests/unit/caching/test_request_redis_batch_pre_call.py (file lacks "router" in name) + "_deployment_params_base_model", # Tested through log_retry in test_router_helper_utils.py "_configured_model_info", # Tested through get_configured_service_tiers in test_router.py "_routable_deployments", # Tested through get_configured_service_tiers and get_routable_upstream_model in test_router.py "_async_get_available_deployment", # Body of the `route {model}` phase wrapper, exercised through async_get_available_deployment in test_router.py diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 960bbc9cba3..7b18cb3350c 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -2347,6 +2347,74 @@ def test_every_earlier_timed_out_attempt_is_billed_at_its_own_deployment_rate(mo assert all(a.usage is usage for a in attempts) +def test_an_earlier_attempt_on_another_model_is_priced_at_that_models_rate_not_the_final_models(monkeypatch): + """After a fallback hop the final attempt runs on another model group, so the records of the group + that timed out first name their own deployment model (and its ``base_model`` when the deployment + name is not in the cost map). Each is billed at that model's input rate, not at the final model's.""" + from litellm.proxy.utils import _billed_earlier_attempts + + for pricing_model, provider, input_rate in ( + ("lit145-final-model", "openai", 0.5), + ("lit145-first-model", "openai", 0.02), + ("azure/lit145-first-model", "azure", 0.02), + ): + monkeypatch.setitem( + litellm.model_cost, + pricing_model, + {"litellm_provider": provider, "mode": "chat", "input_cost_per_token": input_rate, "output_cost_per_token": 1.0}, + ) + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + + attempts = _billed_earlier_attempts( + model_call_details={"model": "openai/lit145-final-model", "custom_llm_provider": "openai"}, + litellm_params={ + "metadata": { + "attempted_retries": 1, + "model_info": {"id": "dep-lit145-final"}, + "previous_models": ( + { + "deployment_id": "dep-lit145-first", + "model": "openai/lit145-first-model", + "base_model": None, + "exception_type": "Timeout", + "attempted_retries": 0, + }, + { + "deployment_id": "dep-lit145-first-azure", + "model": "azure/lit145-first-deployment", + "base_model": "azure/lit145-first-model", + "exception_type": "Timeout", + "attempted_retries": 1, + }, + { + "deployment_id": "dep-lit145-final", + "model": "openai/lit145-final-model", + "base_model": None, + "exception_type": "Timeout", + "attempted_retries": 0, + }, + { + "deployment_id": "dep-lit145-final", + "model": "openai/lit145-final-model", + "base_model": None, + "exception_type": "Timeout", + "attempted_retries": 1, + }, + ), + } + }, + estimated_usage=usage, + base_model=None, + deployment_id="dep-lit145-final", + ) + + assert [(a.attempted_retries, a.deployment_id, a.response_cost) for a in attempts] == [ + (0, "dep-lit145-first", pytest.approx(24 * 0.02)), + (1, "dep-lit145-first-azure", pytest.approx(24 * 0.02)), + (0, "dep-lit145-final", pytest.approx(24 * 0.5)), + ] + + def test_earlier_attempts_are_priced_from_the_prompt_estimate_not_the_final_streams_recovered_usage(): """A final attempt whose stream broke mid-flight lifts its recovered usage (prompt plus the chunks delivered); the earlier timed-out attempts never produced output, so they are priced from the diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index d645ab4566d..13ac55fa937 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -15195,7 +15195,7 @@ async def test_cyclic_fallback_graph_does_not_amplify_one_request(): _FLAT_ATTEMPT_RECORD_KEYS = frozenset( - {"model_group", "deployment_id", "exception_type", "exception_string", "attempted_retries"} + {"model_group", "deployment_id", "model", "base_model", "exception_type", "exception_string", "attempted_retries"} ) _BREADCRUMB_CREDENTIAL_CANARY = "Bearer sk-ant-oat01-RETRY-BREADCRUMB-CANARY-doNotShip" diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index e3061294bef..7088818aa01 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -837,7 +837,12 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) -> "model": "gpt-3.5-turbo", "api_key": "sk-must-not-be-recorded", "messages": [{"role": "user", "content": "hi"}], - metadata_key: {"model_info": {"id": "deployment-1"}, "attempted_retries": 2, "user_api_key": "sk-proxy"}, + metadata_key: { + "model_info": {"id": "deployment-1", "base_model": "azure/gpt-3.5-turbo"}, + "deployment": "azure/my-35-deployment", + "attempted_retries": 2, + "user_api_key": "sk-proxy", + }, }, e=rate_limit_error, ) @@ -845,6 +850,8 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) -> { "model_group": "gpt-3.5-turbo", "deployment_id": "deployment-1", + "model": "azure/my-35-deployment", + "base_model": "azure/gpt-3.5-turbo", "exception_type": "RateLimitError", "exception_string": "litellm.RateLimitError: slow down", "attempted_retries": 2, @@ -856,6 +863,38 @@ def test_log_retry(model_list: list[DeploymentTypedDict], metadata_key: str) -> assert router.log_retry(kwargs=planted_kwargs, e=rate_limit_error)[metadata_key]["request_retry_count"] == 1 +def test_log_retry_reads_base_model_from_the_deployments_litellm_params() -> None: + """An Azure named deployment declares ``base_model`` under ``litellm_params``, not ``model_info``, and + that is the model the failure row is priced at, so the retry record has to carry it too""" + router = Router( + model_list=[ + { + "model_name": "azure-group", + "litellm_params": { + "model": "azure/my-named-deployment", + "api_key": "sk-fake", + "api_base": "https://example.openai.azure.com", + "base_model": "azure/gpt-4o-mini", + }, + "model_info": {"id": "dep-azure-named"}, + } + ] + ) + new_kwargs = router.log_retry( + kwargs={ + "model": "azure-group", + "metadata": { + "model_info": {"id": "dep-azure-named"}, + "deployment": "azure/my-named-deployment", + "attempted_retries": 0, + }, + }, + e=litellm.Timeout(message="slow", llm_provider="azure", model="azure/my-named-deployment"), + ) + record = new_kwargs["metadata"]["previous_models"][0] + assert (record["model"], record["base_model"]) == ("azure/my-named-deployment", "azure/gpt-4o-mini") + + @pytest.mark.usefixtures("router_minute_pinned") def test_update_usage(model_list): """Test if the '_update_usage' function is working correctly""" From 6b1f4ed46f5fd81ed55a49ba61563587acc72bd7 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 15:28:19 -0700 Subject: [PATCH 012/235] fix(logging): spilled over PTU requests without base_model log the served model's rates (#47655) * fix(logging): report pay-as-you-go rates for spilled over PTU requests without base_model * fix(logging): keep the provider response model so spilled over PTU payloads price the served model * fix(logging): keep a provider_response_model the response already carries * fix(logging): read the carried provider_response_model through a typed adapter --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 30 ++++++----- .../llm_response_utils/response_metadata.py | 29 +++++++++++ .../test_response_metadata.py | 36 +++++++++++++ .../test_litellm_logging.py | 51 ++++++++++++++++++- 4 files changed, 132 insertions(+), 14 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 93b45be4922..004dd7b65a5 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2209,20 +2209,23 @@ class Logging(LiteLLMLoggingBaseClass): usage=usage, ) - def custom_pricing_for(self, result: object) -> bool: - litellm_params: Final = getattr(self, "litellm_params", None) + def spilled_over_ptu_for(self, result: object) -> bool: result_hidden_params: Final = getattr(result, HIDDEN_PARAMS_ATTR, None) or MappingProxyType({}) additional_headers: Final = ( result_hidden_params.get("additional_headers") if isinstance(result_hidden_params, dict) else getattr(result_hidden_params, "additional_headers", None) ) - spilled_over: Final = is_spilled_over_ptu_request( - model_info=_deployment_model_info(litellm_params), + return is_spilled_over_ptu_request( + model_info=_deployment_model_info(getattr(self, "litellm_params", None)), response_headers=self.model_call_details.get("response_headers"), additional_headers=additional_headers, ) - return False if spilled_over else use_custom_pricing_for_model(litellm_params=litellm_params) + + def custom_pricing_for(self, result: object) -> bool: + if self.spilled_over_ptu_for(result): + return False + return use_custom_pricing_for_model(litellm_params=getattr(self, "litellm_params", None)) def _prompt_for_cost_calculation(self) -> str: """ @@ -7245,13 +7248,16 @@ def get_standard_logging_object_payload( saved_cache_cost: Final = (uncached_price.cost if uncached_price is not None else None) or 0.0 ## Get model cost information ## - base_model = get_base_model_from_metadata(model_call_details=kwargs) - # The router overrides completion_response.model to the model-group alias before - # this payload is built, so cost-map lookup via that alias always misses. - # Fall back to the actual deployment model set by the router in metadata. - if base_model is None: - base_model = metadata.get("deployment") - custom_pricing: Final = logging_obj.custom_pricing_for(init_response_obj) + configured_base_model: Final = get_base_model_from_metadata(model_call_details=kwargs) + spilled_over: Final = logging_obj.spilled_over_ptu_for(init_response_obj) + # The router overrides completion_response.model to the model-group alias before this payload is + # built, so cost-map lookup via that alias misses and the deployment model stands in for base_model. + # A PTU request Azure spilled onto pay-as-you-go capacity is billed at the served model's rates, + # which the deployment's zeroed PTU entry would misreport, so that fallback is skipped there. + base_model: Final = ( + metadata.get("deployment") if configured_base_model is None and not spilled_over else configured_base_model + ) + custom_pricing: Final = False if spilled_over else logging_obj.custom_pricing_for(init_response_obj) raw_response_cost: Final = kwargs.get("response_cost") llm_response_cost: Final[float] = raw_response_cost or 0.0 guardrail_cost: Final = guardrail_information_cost(metadata.get("standard_logging_guardrail_information")) diff --git a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py index ff82aacf393..ace57a64aae 100644 --- a/litellm/litellm_core_utils/llm_response_utils/response_metadata.py +++ b/litellm/litellm_core_utils/llm_response_utils/response_metadata.py @@ -4,6 +4,7 @@ from functools import reduce from typing import Final import httpx +from pydantic import TypeAdapter, ValidationError from litellm.constants import LITELLM_DETAILED_TIMING from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs, process_response_headers @@ -101,6 +102,32 @@ def response_timing_metrics( return {"_response_ms": total_response_time_ms, "litellm_overhead_time_ms": overhead_ms} +_HIDDEN_PARAMS_DICT: Final = TypeAdapter(dict[str, object]) + + +def _hidden_params_mapping(hidden_params: object) -> Mapping[str, object]: + try: + return _HIDDEN_PARAMS_DICT.validate_python(hidden_params) + except ValidationError: + return {} + + +def _existing_provider_response_model(result: object) -> str | None: + """The pricing model a response already carries, like the decisions path sets, wins over the response model.""" + hidden_params: Final[object] = getattr(result, "_hidden_params", None) + carried: Final[object] = _hidden_params_mapping(hidden_params).get("provider_response_model") or getattr( + hidden_params, "provider_response_model", None + ) + return carried if isinstance(carried, str) and carried else None + + +def _provider_response_model(result: object) -> str | None: + """The model the provider answered with, kept because the proxy later rewrites ``result.model`` to the alias the + client asked for and cost-map lookups for the standard logging payload would otherwise miss.""" + model: Final = getattr(result, "model", None) if isinstance(result, ModelResponse) else None + return model if isinstance(model, str) and model else None + + class ResponseMetadata: """ Handles setting and managing `_hidden_params`, `response_time_ms`, and `litellm_overhead_time_ms` for LiteLLM responses @@ -147,6 +174,8 @@ class ResponseMetadata: preserve_litellm_internal_headers=True, ), "litellm_model_name": model, + "provider_response_model": _existing_provider_response_model(self.result) + or _provider_response_model(self.result), } self._update_hidden_params(new_params) diff --git a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py index 00b0e2f6972..6d9d8c7997a 100644 --- a/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py +++ b/tests/unit/litellm_core_utils/llm_response_utils/test_response_metadata.py @@ -17,6 +17,7 @@ import pytest import litellm import litellm.litellm_core_utils.llm_response_utils.response_metadata as response_metadata_mod import litellm.proxy.common_request_processing as common_request_processing_mod +from litellm.cost_calculator import select_model_name_for_cost_calc from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( ResponseMetadata, @@ -649,3 +650,38 @@ def test_update_response_metadata_prices_per_second_deployment_from_its_stamped_ assert result._response_ms == pytest.approx(2000) assert result._hidden_params["response_cost"] == pytest.approx(0.02 * 2) + + +def test_update_response_metadata_keeps_the_provider_model_after_the_proxy_rewrites_response_model(): + start_time: Final = datetime.datetime(2026, 9, 21, 12, 0, 0) + logging_obj: Final = Logging( + model="azure/ptu-deployment", + messages=[{"role": "user", "content": "Hello"}], + stream=False, + call_type="completion", + start_time=start_time, + litellm_call_id="provider-model-response-metadata", + function_id="f", + ) + logging_obj.update_environment_variables( + model="azure/ptu-deployment", litellm_params={}, optional_params={}, custom_llm_provider="azure" + ) + result: Final = ModelResponse( + model="gpt-5.4-nano-2026-03-17", + usage=Usage(prompt_tokens=11, completion_tokens=7, total_tokens=18), + ) + + update_response_metadata( + result=result, + logging_obj=logging_obj, + model="azure/ptu-deployment", + kwargs={}, + start_time=start_time, + end_time=start_time + datetime.timedelta(seconds=2), + ) + result.model = "model-group-alias" + + assert ( + select_model_name_for_cost_calc(model=None, completion_response=result, custom_llm_provider="azure") + == "azure/gpt-5.4-nano-2026-03-17" + ) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index b3414ebe9d7..79645ffe8f6 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -10168,6 +10168,7 @@ class TestAzurePTUSpilloverCost: ROUTER_MODEL_ID: Final = "ptu-spill-router-model-id" SERVED_MODEL: Final = "azure/spill-served-model-ptu" + PTU_DEPLOYMENT: Final = "azure/spill-ptu-deployment" PTU_MODEL_INFO: Final = { "id": ROUTER_MODEL_ID, "team_id": "team-1", @@ -10203,7 +10204,9 @@ class TestAzurePTUSpilloverCost: litellm.model_cost.pop(TestAzurePTUSpilloverCost.ROUTER_MODEL_ID, None) litellm.model_cost.pop(TestAzurePTUSpilloverCost.SERVED_MODEL, None) - def _logging_obj(self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch) -> LitellmLogging: + def _logging_obj( + self, model_info: dict, *, flag: str, litellm_rate: float, monkeypatch, deployment: str | None = None + ) -> LitellmLogging: monkeypatch.setenv("LITELLM_ENABLE_PTU_COST_ATTRIBUTION", flag) obj = LitellmLogging( model=self.SERVED_MODEL, @@ -10220,7 +10223,7 @@ class TestAzurePTUSpilloverCost: optional_params={}, litellm_params={ "api_base": "", - "metadata": {"model_info": model_info}, + "metadata": {"model_info": model_info, "deployment": deployment}, "input_cost_per_token": litellm_rate, "output_cost_per_token": litellm_rate, }, @@ -10324,6 +10327,50 @@ class TestAzurePTUSpilloverCost: finally: self._unregister_models() + def test_spillover_without_base_model_reports_the_pay_as_you_go_rates_billing_used(self, monkeypatch) -> None: + self._register_models() + litellm.model_cost[self.PTU_DEPLOYMENT] = {"litellm_provider": "azure", "mode": "chat"} + try: + obj = self._logging_obj( + dict(self.PTU_MODEL_INFO), + flag="True", + litellm_rate=0.0, + monkeypatch=monkeypatch, + deployment=self.PTU_DEPLOYMENT, + ) + response = self._response() + response._hidden_params["additional_headers"] = {"llm_provider-x-ms-is-spilled-over": "true"} + response._hidden_params["provider_response_model"] = response.model + response_cost: Final = obj.response_cost_calculator(result=response) + response.model = "spill-model-group-alias" + + payload: Final = get_standard_logging_object_payload( + kwargs={ + "model": self.SERVED_MODEL, + "custom_llm_provider": "azure", + "litellm_params": obj.litellm_params, + "response_cost": response_cost, + }, + init_response_obj=response, + start_time=datetime.datetime.now(), + end_time=datetime.datetime.now(), + logging_obj=obj, + status="success", + ) + + assert response_cost == pytest.approx(self.EXPECTED_SPILL_COST) + assert payload is not None + model_map_information: Final = payload["model_map_information"] + assert model_map_information["model_map_key"] != self.PTU_DEPLOYMENT + model_map_value: Final = model_map_information["model_map_value"] + assert model_map_value is not None + assert response_cost == pytest.approx( + 100 * model_map_value["input_cost_per_token"] + 50 * model_map_value["output_cost_per_token"] + ) + finally: + litellm.model_cost.pop(self.PTU_DEPLOYMENT, None) + self._unregister_models() + def test_spillover_header_does_not_touch_non_ptu_custom_pricing(self, monkeypatch) -> None: self._register_models() custom_model_id: Final = "non-ptu-custom-router-model-id" From 8e3ce6d110cabfc4b0212ed143dff59d696e31ef Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:34:37 -0700 Subject: [PATCH 013/235] fix(vertex-ai): drop flex prices from gemini-nano-banana-2.1 (#47666) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 ------ model_prices_and_context_window.json | 6 ------ 2 files changed, 12 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 15eba7f8af8..b6521557f83 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -75064,12 +75064,10 @@ "gemini-nano-banana-2.1": { "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, - "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, "input_cost_per_image": 0.00168, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, - "input_cost_per_token_flex": 7.5e-07, "input_cost_per_token_priority": 2.7e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, @@ -75080,7 +75078,6 @@ "output_cost_per_image_token": 3e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, - "output_cost_per_token_flex": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -75113,12 +75110,10 @@ "vertex_ai/gemini-nano-banana-2.1": { "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, - "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, "input_cost_per_image": 0.00168, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, - "input_cost_per_token_flex": 7.5e-07, "input_cost_per_token_priority": 2.7e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, @@ -75129,7 +75124,6 @@ "output_cost_per_image_token": 3e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, - "output_cost_per_token_flex": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 15eba7f8af8..b6521557f83 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -75064,12 +75064,10 @@ "gemini-nano-banana-2.1": { "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, - "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, "input_cost_per_image": 0.00168, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, - "input_cost_per_token_flex": 7.5e-07, "input_cost_per_token_priority": 2.7e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, @@ -75080,7 +75078,6 @@ "output_cost_per_image_token": 3e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, - "output_cost_per_token_flex": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", @@ -75113,12 +75110,10 @@ "vertex_ai/gemini-nano-banana-2.1": { "cache_read_input_token_cost": 1.5e-07, "cache_read_input_token_cost_batches": 7.5e-08, - "cache_read_input_token_cost_flex": 7.5e-08, "cache_read_input_token_cost_priority": 2.7e-07, "input_cost_per_image": 0.00168, "input_cost_per_token": 1.5e-06, "input_cost_per_token_batches": 7.5e-07, - "input_cost_per_token_flex": 7.5e-07, "input_cost_per_token_priority": 2.7e-06, "litellm_provider": "vertex_ai-language-models", "max_input_tokens": 131072, @@ -75129,7 +75124,6 @@ "output_cost_per_image_token": 3e-05, "output_cost_per_token": 7.5e-06, "output_cost_per_token_batches": 3.75e-06, - "output_cost_per_token_flex": 3.75e-06, "output_cost_per_token_priority": 1.35e-05, "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014}, "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing", From 5c836d6e13d73407e9c35dc4b7da1dcceca73665 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 15:36:29 -0700 Subject: [PATCH 014/235] feat(cost-map): add fireworks deepseek v4.1 flash 1017 us at the us-only premium (#47667) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 20 +++++++++++++++++++ model_prices_and_context_window.json | 20 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b6521557f83..c6aea8ec98a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76923,5 +76923,25 @@ "supported_output_modalities": [ "text" ] + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash-1017-us": { + "cache_read_input_token_cost": 9e-09, + "cache_read_input_token_cost_priority": 1.125e-08, + "input_cost_per_token": 4.5e-07, + "input_cost_per_token_priority": 5.625e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.8e-06, + "output_cost_per_token_priority": 2.25e-06, + "source": "https://docs.fireworks.ai/serverless/us-only-serverless", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b6521557f83..c6aea8ec98a 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76923,5 +76923,25 @@ "supported_output_modalities": [ "text" ] + }, + "fireworks_ai/accounts/fireworks/models/deepseek-v4p1-flash-1017-us": { + "cache_read_input_token_cost": 9e-09, + "cache_read_input_token_cost_priority": 1.125e-08, + "input_cost_per_token": 4.5e-07, + "input_cost_per_token_priority": 5.625e-07, + "litellm_provider": "fireworks_ai", + "max_input_tokens": 1048576, + "max_output_tokens": 393216, + "max_tokens": 393216, + "mode": "chat", + "output_cost_per_token": 1.8e-06, + "output_cost_per_token_priority": 2.25e-06, + "source": "https://docs.fireworks.ai/serverless/us-only-serverless", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true } } From 9717b38050e25b63b5fa9c70201475242b20e0f0 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 20 Oct 2026 15:37:26 -0700 Subject: [PATCH 015/235] bump: litellm-enterprise 0.1.78 -> 0.1.79, litellm-proxy-extras 0.4.110 -> 0.4.111, litellm 1.107.0 -> 1.108.0 (#47668) --- enterprise/pyproject.toml | 4 ++-- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 8 ++++---- uv.lock | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 63f6979ca3d..17209a29e15 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.78" +version = "0.1.79" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.78" +version = "0.1.79" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index a1dee5ac671..6de44be0c12 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.110" +version = "0.4.111" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.110" +version = "0.4.111" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index 9871a83b421..4cb529bbdb3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm" -version = "1.107.0" +version = "1.108.0" description = "Library to easily interface with LLM API providers" readme = "README.md" requires-python = ">=3.10, <3.15" @@ -78,8 +78,8 @@ proxy = [ "mcp>=2.2.0,<3", "httpx2>=2.5.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.110", - "litellm-enterprise==0.1.78", + "litellm-proxy-extras==0.4.111", + "litellm-enterprise==0.1.79", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", @@ -376,7 +376,7 @@ litellm-enterprise = { workspace = true } members = ["enterprise", "litellm-proxy-extras"] [tool.commitizen] -version = "1.107.0" +version = "1.108.0" version_files = [ "pyproject.toml:^version", ] diff --git a/uv.lock b/uv.lock index 9b490cf95f4..297344d1f7a 100644 --- a/uv.lock +++ b/uv.lock @@ -4603,7 +4603,7 @@ wheels = [ [[package]] name = "litellm" -version = "1.107.0" +version = "1.108.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -5105,12 +5105,12 @@ provides-extras = ["test"] [[package]] name = "litellm-enterprise" -version = "0.1.78" +version = "0.1.79" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.110" +version = "0.4.111" source = { editable = "litellm-proxy-extras" } dependencies = [ { name = "psycopg" }, From 9fbeebc9883549aa5036c48b84a2985fed6b9ab1 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 22:51:45 +0000 Subject: [PATCH 016/235] feat(extraction): add /v1/extraction with Cloudflare Clairic support (#47473) * feat(extraction): add Clairic extraction types and Cloudflare provider config * feat(extraction): add the /v1/extraction route, router wiring, and Clairic billing * fix(extraction): refuse or drop stream, pass unknown keys, run guardrails, keep the requested model name Clairic does not stream, so stream=true answers 400 and is dropped and billed under drop_params. Keys LiteLLM does not know pass through to Cloudflare. Guardrails scan state, field instructions and text answers. Frozen decision and extraction responses now carry the requested model name. List fields must name their item type. /health/test_connection and the Add Model page accept the extraction mode. * fix(extraction): keep upstream model names on decisions, scan images, keep typed values, refuse OpenAI params Scope the frozen-response model stamp to extraction so /v1/decisions and /v1/systemone return the upstream name as on main. Pass request images to guardrails as data URLs and drop answer spans when a response guardrail rewrites state_text. Field spec values keep their JSON type, so a wrong type answers 400 instead of being coerced. OpenAI generation params answer 400 unless drop_params. The Model Hub shows a /v1/extraction snippet for extraction models. * fix(extraction): send only Clairic's request fields, count extraction tokens, keep typed answers typed, drop spans a request guardrail made stale, and honor --max_tokens * fix(extraction): scan number answers in response guardrails, rewrite guardrail fields in linear time, and leave --max_tokens as on main * fix(extraction): scan float answers as whole numbers so fraction digits never trip a response guardrail * fix(extraction): round only mean and median for response guardrails so fractional values keep their decimal text * fix(extraction): count state and fields for budget reservations and scan estimate quantiles in response guardrails * fix(extraction): send each repeated text to guardrails once and keep the context-window budget reservation * test(extraction): type request_data in the guardrail test helpers --- .github/workflows/test-unit.yml | 1 + gateway/routes/allowlist.py | 4 +- litellm/__init__.py | 4 + litellm/_lazy_imports_registry.py | 2 + litellm/cost_calculator.py | 21 +- litellm/extraction/__init__.py | 3 + litellm/extraction/main.py | 351 ++++++ .../health_check_helpers.py | 20 +- .../litellm_core_utils/health_check_utils.py | 2 + litellm/litellm_core_utils/litellm_logging.py | 2 + litellm/llms/base_llm/extraction/__init__.py | 3 + .../base_llm/extraction/transformation.py | 86 ++ litellm/llms/cloudflare/common_utils.py | 53 + .../cloudflare/decisions/transformation.py | 48 +- .../guardrail_translation/__init__.py | 11 + .../guardrail_translation/handler.py | 276 +++++ .../cloudflare/extraction/transformation.py | 32 + litellm/llms/custom_httpx/llm_http_handler.py | 112 ++ ...odel_prices_and_context_window_backup.json | 18 + litellm/proxy/_lazy_features.py | 2 +- litellm/proxy/_lazy_openapi_snapshot.json | 1037 +++++++++++++++++ litellm/proxy/_types.py | 2 + .../auth/managed_authorization.py | 2 + litellm/proxy/common_request_processing.py | 29 +- .../proxy/decisions_endpoints/endpoints.py | 103 +- .../health_endpoints/_health_endpoints.py | 1 + .../hooks/parallel_request_limiter_v3.py | 56 +- litellm/proxy/route_llm_request.py | 2 + litellm/router.py | 9 + litellm/types/extraction.py | 162 +++ litellm/types/utils.py | 7 + litellm/utils.py | 24 +- model_prices_and_context_window.json | 18 + model_prices_and_context_window.schema.json | 1 + .../test_extraction_guardrail_benchmarks.py | 61 + tests/unit/extraction/__init__.py | 0 tests/unit/extraction/test_main.py | 422 +++++++ .../test_health_check_helpers.py | 96 +- .../llms/azure/test_azure_common_utils.py | 1 + ...cloudflare_extraction_guardrail_handler.py | 335 ++++++ ...st_cloudflare_extraction_transformation.py | 221 ++++ .../decisions_endpoints/test_endpoints.py | 329 +++++- .../health_endpoints/test_health_endpoints.py | 62 +- .../hooks/test_parallel_request_limiter_v3.py | 76 ++ .../proxy/test_common_request_processing.py | 24 + tests/unit/test_utils.py | 3 +- .../AIHub/modelUsageExample.test.ts | 11 + .../src/components/AIHub/modelUsageExample.ts | 27 +- .../AddModelForm.integration.test.tsx | 4 +- .../components/add_model/add_model_modes.tsx | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 481 +++++++- 51 files changed, 4510 insertions(+), 148 deletions(-) create mode 100644 litellm/extraction/__init__.py create mode 100644 litellm/extraction/main.py create mode 100644 litellm/llms/base_llm/extraction/__init__.py create mode 100644 litellm/llms/base_llm/extraction/transformation.py create mode 100644 litellm/llms/cloudflare/common_utils.py create mode 100644 litellm/llms/cloudflare/extraction/guardrail_translation/__init__.py create mode 100644 litellm/llms/cloudflare/extraction/guardrail_translation/handler.py create mode 100644 litellm/llms/cloudflare/extraction/transformation.py create mode 100644 litellm/types/extraction.py create mode 100644 tests/benchmarks/test_extraction_guardrail_benchmarks.py create mode 100644 tests/unit/extraction/__init__.py create mode 100644 tests/unit/extraction/test_main.py create mode 100644 tests/unit/llms/cloudflare/test_cloudflare_extraction_guardrail_handler.py create mode 100644 tests/unit/llms/cloudflare/test_cloudflare_extraction_transformation.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2b9263f0430..c593acbe8fb 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -139,6 +139,7 @@ jobs: test-path: |- tests/unit/batch_completion tests/unit/decisions + tests/unit/extraction tests/unit/litellm_core_utils tests/unit/systemone python-version: "3.12" diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index cced49f788a..cc87a2ed16b 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -1,7 +1,7 @@ """Path allowlist for the gateway component. The gateway exposes the LLM data-plane surface: chat/completions, embeddings, -audio, batches, files, fine-tuning, rerank, decisions, ocr, rag, video, search, image, +audio, batches, files, fine-tuning, rerank, decisions, extraction, ocr, rag, video, search, image, responses, vector stores, passthrough providers, realtime websockets, MCP tool-call endpoints, and operational endpoints (/health, /metrics, and the /debug/memory/summary read of the serving worker's RSS). @@ -64,6 +64,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/decisions", "/v1/systemone", "/systemone", + "/v1/extraction", + "/extraction", "/v1/ocr", "/ocr", "/v1/rag/", diff --git a/litellm/__init__.py b/litellm/__init__.py index f6604a3b4e5..1784deeb797 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1505,6 +1505,7 @@ from .rag.main import * from .sandbox.main import * from .decisions.main import * from .systemone.main import * +from .extraction.main import * from .tool_loop import ToolLoopMaxRoundsExceeded, arun_tool_loop, run_tool_loop from .search.main import * from .realtime_api.main import ( @@ -1711,6 +1712,9 @@ if TYPE_CHECKING: from .llms.cloudflare.decisions.transformation import ( CloudflareDecisionsConfig as CloudflareDecisionsConfig, ) + from .llms.cloudflare.extraction.transformation import ( + CloudflareExtractionConfig as CloudflareExtractionConfig, + ) from .llms.strands_decider.decisions.transformation import ( StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 8abc24d1417..e0cb7c680bc 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -166,6 +166,7 @@ LLM_CONFIG_NAMES: Final = ( "OpenRouterDecisionsConfig", "FireworksAIDecisionsConfig", "CloudflareDecisionsConfig", + "CloudflareExtractionConfig", "StrandsDeciderDecisionsConfig", "DatabricksDecisionsConfig", "HostedVLLMDecisionsConfig", @@ -735,6 +736,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), "FireworksAIDecisionsConfig": (".llms.fireworks_ai.decisions.transformation", "FireworksAIDecisionsConfig"), "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), + "CloudflareExtractionConfig": (".llms.cloudflare.extraction.transformation", "CloudflareExtractionConfig"), "StrandsDeciderDecisionsConfig": ( ".llms.strands_decider.decisions.transformation", "StrandsDeciderDecisionsConfig", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 67178c8959c..f952883bc87 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -112,6 +112,7 @@ from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_toke from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage +from litellm.types.extraction import ExtractionResponse, ExtractionUsage from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel from litellm.types.llms.openai import ( HttpxBinaryResponseContent, @@ -1171,6 +1172,8 @@ def get_usage_object( if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)): return None if completion_response.usage is None else _decisions_usage(completion_response.usage) + if isinstance(completion_response, ExtractionResponse): + return _extraction_usage(completion_response.usage) if isinstance(completion_response, HttpxBinaryResponseContent): return completion_response.usage if usage_obj is None: @@ -1227,6 +1230,14 @@ def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage: ) +def _extraction_usage(usage: ExtractionUsage) -> Usage: + return Usage( + prompt_tokens=usage.input_tokens, + completion_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + ) + + def _is_known_usage_objects(usage_obj): """Returns True if the usage obj is a known Usage type""" return ( @@ -1234,6 +1245,7 @@ def _is_known_usage_objects(usage_obj): or isinstance(usage_obj, ResponseAPIUsage) or isinstance(usage_obj, DecisionsUsage) or isinstance(usage_obj, OpenAIDecisionUsage) + or isinstance(usage_obj, ExtractionUsage) or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj) ) @@ -1263,6 +1275,8 @@ def infer_call_type(call_type: CallTypesLiteral | None, completion_response: obj return "send_message" elif isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)): return "adecisions" + elif isinstance(completion_response, ExtractionResponse): + return "aextraction" return call_type @@ -1674,6 +1688,8 @@ def completion_cost( ) if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)): _usage = _decisions_usage(usage_obj).model_dump() + elif isinstance(usage_obj, ExtractionUsage): + _usage = _extraction_usage(usage_obj).model_dump() elif usage_obj is None: _usage = {} elif isinstance(usage_obj, BaseModel): @@ -2199,7 +2215,8 @@ def response_cost_calculator( | Response | SearchResponse | DecisionsResponse - | OpenAIDecisionResponse, + | OpenAIDecisionResponse + | ExtractionResponse, model: str, custom_llm_provider: str | None, call_type: Literal[ @@ -2225,6 +2242,8 @@ def response_cost_calculator( "adecisions", "systemone", "asystemone", + "extraction", + "aextraction", ], optional_params: dict, cache_hit: bool | None = None, diff --git a/litellm/extraction/__init__.py b/litellm/extraction/__init__.py new file mode 100644 index 00000000000..bdacdae6ea4 --- /dev/null +++ b/litellm/extraction/__init__.py @@ -0,0 +1,3 @@ +from litellm.extraction.main import aextraction, extraction + +__all__ = ["aextraction", "extraction"] diff --git a/litellm/extraction/main.py b/litellm/extraction/main.py new file mode 100644 index 00000000000..94198e911ef --- /dev/null +++ b/litellm/extraction/main.py @@ -0,0 +1,351 @@ +import asyncio +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from functools import partial +from types import MappingProxyType +from typing import Final, TypeAlias + +import httpx +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.decisions.call import log_caller_request +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.tool_fields import DropPolicy, drop_policy +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.types.extraction import ( + EXTRACTION_STATE_REWRITTEN_METADATA_KEY, + ExtractionAnswer, + ExtractionBase64Image, + ExtractionField, + ExtractionOptions, + ExtractionRequestBody, + ExtractionResponse, +) +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders, is_litellm_owned_kwarg +from litellm.utils import ( + ProviderConfigManager, + client, # pyright: ignore[reportUnknownVariableType] # client is an untyped decorator + get_model_cost_key, +) + +ExtractionFields: TypeAlias = Mapping[str, ExtractionField | Mapping[str, object]] +ExtractionImages: TypeAlias = Sequence[str | ExtractionBase64Image | Mapping[str, object]] +ExtractionOptionsParam: TypeAlias = ExtractionOptions | Mapping[str, object] + +_REQUEST_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody) +_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_HANDLER: Final = BaseLLMHTTPHandler() +_LOCAL_KWARGS: Final = frozenset( + ( + *GenericLiteLLMParams.model_fields, + "extra_headers", + "base_url", + "default_headers", + "deployment_id", + "additional_drop_params", + ) +) + + +@dataclass(frozen=True, slots=True, repr=False) +class _ExtractionCall: + model: str + requested_model: str + custom_llm_provider: str + provider_config: BaseExtractionConfig + body: Mapping[str, object] + api_base: str + api_key: str | None + logging_obj: LiteLLMLoggingObj | None + headers: Mapping[str, str] + timeout: float | httpx.Timeout | None + litellm_params: Mapping[str, object] + client: HTTPHandler | AsyncHTTPHandler | None + + +def supported_extraction_providers() -> tuple[str, ...]: + return tuple( + provider.value + for provider in LlmProviders + if ProviderConfigManager.get_provider_extraction_config(model="", provider=provider) is not None + ) + + +def _provider_config(model: str, custom_llm_provider: str) -> BaseExtractionConfig: + provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None) + provider_config: Final = ( + None if provider is None else ProviderConfigManager.get_provider_extraction_config(model, provider) + ) + if provider_config is None: + raise litellm.BadRequestError( + message=( + f"Unknown Extraction provider '{custom_llm_provider}'. " + f"Supported providers: {', '.join(supported_extraction_providers())}" + ), + model=model, + llm_provider=custom_llm_provider, + ) + return provider_config + + +def _request_fields( + *, + state: str | None, + fields: ExtractionFields | None, + images: ExtractionImages | None, + reference_date: str | None, + options: ExtractionOptionsParam | None, +) -> Mapping[str, object]: + sent: Final[Mapping[str, object | None]] = MappingProxyType( + {"state": state, "fields": fields, "images": images, "reference_date": reference_date, "options": options} + ) + return MappingProxyType({name: value for name, value in sent.items() if value is not None}) + + +def _is_unsupported(name: str, value: object, policy: DropPolicy) -> bool: + return ( + value is not None + and value is not False + and name not in _LOCAL_KWARGS + and not is_litellm_owned_kwarg(name) + and not policy.drops(name) + ) + + +def _unsupported_params(kwargs: Mapping[str, object]) -> tuple[str, ...]: + policy: Final = drop_policy(kwargs) + return tuple(sorted(name for name, value in kwargs.items() if _is_unsupported(name, value, policy))) + + +def _prepare_call( + *, + model: str, + request_fields: Mapping[str, object], + api_key: str | None, + api_base: str | None, + timeout: float | httpx.Timeout | None, + custom_llm_provider: str | None, + extra_headers: Mapping[str, str] | None, + kwargs: Mapping[str, object], +) -> _ExtractionCall: + log_caller_request(kwargs, request_fields) + upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + provider_config: Final = _provider_config(model, provider) + unsupported: Final = _unsupported_params(kwargs) + if unsupported: + raise litellm.UnsupportedParamsError( + message=( + f"{provider} does not support parameters: {list(unsupported)}, for model={model}. " + "To drop these, set `litellm.drop_params=True` or for proxy:\n\n" + "`litellm_settings:\n drop_params: true`\n" + ), + model=model, + llm_provider=provider, + ) + try: + request: Final = _REQUEST_ADAPTER.validate_python(request_fields) + except ValidationError as error: + raise litellm.BadRequestError( + message=f"Invalid Extraction request: {error}", model=model, llm_provider=provider + ) from error + canonical_model: Final = provider_config.canonical_model(upstream_model) + resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base) + if resolved_api_base is None: + raise litellm.BadRequestError( + message=provider_config.missing_api_base_message(provider), model=model, llm_provider=provider + ) + resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key) + if resolved_api_key is None: + raise litellm.BadRequestError( + message=f"Missing API key for Extraction provider '{provider}'", model=model, llm_provider=provider + ) + logging_obj: Final = kwargs.get("litellm_logging_obj") + http_client: Final = kwargs.get("client") + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_from_kwargs( # pyright: ignore[reportUnknownMemberType] # Logging takes untyped dicts + kwargs=dict(kwargs), + model=canonical_model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), + }, + custom_llm_provider=provider, + ) + return _ExtractionCall( + model=canonical_model, + requested_model=model, + custom_llm_provider=provider, + provider_config=provider_config, + body=provider_config.transform_extraction_request(canonical_model, request), + api_base=resolved_api_base, + api_key=resolved_api_key, + logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + headers=MappingProxyType(dict(extra_headers or {})), + timeout=timeout, + litellm_params=kwargs, + client=http_client if isinstance(http_client, HTTPHandler | AsyncHTTPHandler) else None, + ) + + +def _state_rewritten_by_guardrail(litellm_params: Mapping[str, object]) -> bool: + metadata: Final = litellm_params.get(get_metadata_variable_name_from_kwargs(litellm_params)) + return ( + isinstance(metadata, Mapping) + and _METADATA_ADAPTER.validate_python(metadata).get(EXTRACTION_STATE_REWRITTEN_METADATA_KEY) is True + ) + + +def _without_spans(answers: Mapping[str, ExtractionAnswer]) -> Mapping[str, ExtractionAnswer]: + return {name: answer.model_copy(update={"span": None, "spans": None}) for name, answer in answers.items()} + + +def _formatted_response(response: ExtractionResponse, call: _ExtractionCall) -> ExtractionResponse: + spans_index_hidden_text: Final = response.state_text is None and _state_rewritten_by_guardrail(call.litellm_params) + formatted: Final = response.model_copy( + update={ + "model": call.requested_model, + **({"answers": _without_spans(response.answers)} if spans_index_hidden_text else {}), + } + ) + formatted.set_hidden_params( + { + "model": f"{call.custom_llm_provider}/{call.model}", + "custom_llm_provider": call.custom_llm_provider, + "provider_response_model": ( + get_model_cost_key(f"{call.custom_llm_provider}/{response.model}") + or f"{call.custom_llm_provider}/{call.model}" + ), + } + ) + return formatted + + +def _upstream_exception(error: Exception, call: _ExtractionCall) -> Exception: + if isinstance(error, BaseLLMException) and error.status_code_is_synthesized: + return litellm.APIConnectionError( + message=f"{call.custom_llm_provider.capitalize()}Exception - {error.message}", + llm_provider=call.custom_llm_provider, + model=f"{call.custom_llm_provider}/{call.model}", + ) + return litellm.exception_type( # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # untyped + model=f"{call.custom_llm_provider}/{call.model}", + custom_llm_provider=call.custom_llm_provider, + original_exception=error, + ) + + +async def _asend(call: _ExtractionCall) -> ExtractionResponse: + try: + response: Final = await _HANDLER.aextraction( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + client=call.client if isinstance(call.client, AsyncHTTPHandler) else None, + ) + except Exception as error: + raise _upstream_exception(error, call) from error + return _formatted_response(response, call) + + +def _send(call: _ExtractionCall) -> ExtractionResponse: + try: + response: Final = _HANDLER.extraction( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + body=call.body, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, + litellm_params=call.litellm_params, + client=call.client if isinstance(call.client, HTTPHandler) else None, + ) + except Exception as error: + raise _upstream_exception(error, call) from error + return _formatted_response(response, call) + + +@client +async def aextraction( + model: str, + fields: ExtractionFields, + state: str | None = None, + images: ExtractionImages | None = None, + reference_date: str | None = None, + options: ExtractionOptionsParam | None = None, + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + **kwargs: object, # kwargs-ok: litellm_params forwarded by @client +) -> ExtractionResponse: + call: Final = await asyncio.to_thread( + partial( + _prepare_call, + model=model, + request_fields=_request_fields( + state=state, fields=fields, images=images, reference_date=reference_date, options=options + ), + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) + ) + return await _asend(call) + + +@client +def extraction( + model: str, + fields: ExtractionFields, + state: str | None = None, + images: ExtractionImages | None = None, + reference_date: str | None = None, + options: ExtractionOptionsParam | None = None, + api_key: str | None = None, + api_base: str | None = None, + timeout: float | httpx.Timeout | None = None, + custom_llm_provider: str | None = None, + extra_headers: Mapping[str, str] | None = None, + **kwargs: object, # kwargs-ok: litellm_params forwarded by @client +) -> ExtractionResponse: + call: Final = _prepare_call( + model=model, + request_fields=_request_fields( + state=state, fields=fields, images=images, reference_date=reference_date, options=options + ), + api_key=api_key, + api_base=api_base, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + kwargs=kwargs, + ) + return _send(call) + + +__all__ = ["aextraction", "extraction"] diff --git a/litellm/litellm_core_utils/health_check_helpers.py b/litellm/litellm_core_utils/health_check_helpers.py index 03af9076c79..b19bcd941fa 100644 --- a/litellm/litellm_core_utils/health_check_helpers.py +++ b/litellm/litellm_core_utils/health_check_helpers.py @@ -4,6 +4,7 @@ Helper functions for health check calls. import base64 from collections.abc import Awaitable, Callable, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal from litellm.llms.base_llm.ocr.transformation import DocumentType @@ -25,6 +26,9 @@ IMAGE_EDIT_HEALTH_CHECK_PROMPT: Final = ( ) ANTHROPIC_MESSAGES_HEALTH_CHECK_MAX_TOKENS: Final = 16 +EXTRACTION_HEALTH_CHECK_FIELDS: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType( + {"reachable": MappingProxyType({"type": "text", "instructions": "Any word from the text"})} +) def native_health_check_mode(model: str, custom_llm_provider: str | None) -> Literal["anthropic_messages"] | None: @@ -215,6 +219,7 @@ class HealthCheckHelpers: "anthropic_messages", "ocr", "decisions", + "extraction", ], Callable, ]: @@ -237,7 +242,11 @@ class HealthCheckHelpers: from litellm.litellm_core_utils.audio_utils.utils import ( get_audio_file_for_health_check, ) - from litellm.litellm_core_utils.health_check_utils import DECISIONS_CALL_PARAMS, _filter_model_params + from litellm.litellm_core_utils.health_check_utils import ( + DECISIONS_CALL_PARAMS, + EXTRACTION_CALL_PARAMS, + _filter_model_params, + ) from litellm.realtime_api.main import realtime_health_check return { @@ -322,4 +331,13 @@ class HealthCheckHelpers: } ) ), + "extraction": lambda: litellm.aextraction( + **EXTRACTION_CALL_PARAMS.validate_python( + { + "state": prompt or "health check", + "fields": EXTRACTION_HEALTH_CHECK_FIELDS, + **_filter_model_params(model_params=model_params), + } + ) + ), } diff --git a/litellm/litellm_core_utils/health_check_utils.py b/litellm/litellm_core_utils/health_check_utils.py index 9b344d08d4b..74b6f6dea5d 100644 --- a/litellm/litellm_core_utils/health_check_utils.py +++ b/litellm/litellm_core_utils/health_check_utils.py @@ -8,8 +8,10 @@ from typing import Final from pydantic import TypeAdapter from litellm.types.decisions import DecisionsCallParams +from litellm.types.extraction import ExtractionCallParams DECISIONS_CALL_PARAMS: Final[TypeAdapter[DecisionsCallParams]] = TypeAdapter(DecisionsCallParams) +EXTRACTION_CALL_PARAMS: Final[TypeAdapter[ExtractionCallParams]] = TypeAdapter(ExtractionCallParams) OPTIONAL_STR: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 004dd7b65a5..473666e5ba1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -132,6 +132,7 @@ from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse +from litellm.types.extraction import ExtractionResponse from litellm.types.integrations.s3_v2 import S3PartitionGranularity from litellm.types.interactions import ( InteractionsAPIResponse, @@ -2922,6 +2923,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, SearchResponse) # Search API or isinstance(logging_result, DecisionsResponse) or isinstance(logging_result, OpenAIDecisionResponse) + or isinstance(logging_result, ExtractionResponse) or ( isinstance(logging_result, InteractionsAPIResponse) and logging_result.usage is not None diff --git a/litellm/llms/base_llm/extraction/__init__.py b/litellm/llms/base_llm/extraction/__init__.py new file mode 100644 index 00000000000..fe48996e8b4 --- /dev/null +++ b/litellm/llms/base_llm/extraction/__init__.py @@ -0,0 +1,3 @@ +from .transformation import BaseExtractionConfig + +__all__ = ["BaseExtractionConfig"] diff --git a/litellm/llms/base_llm/extraction/transformation.py b/litellm/llms/base_llm/extraction/transformation.py new file mode 100644 index 00000000000..8461256b883 --- /dev/null +++ b/litellm/llms/base_llm/extraction/transformation.py @@ -0,0 +1,86 @@ +from abc import ABC, abstractmethod +from collections.abc import Mapping +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.extraction import ExtractionRequestBody, ExtractionResponse + +_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) +_RESPONSE_ADAPTER: Final[TypeAdapter[ExtractionResponse]] = TypeAdapter(ExtractionResponse) +_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) + + +class BaseExtractionConfig(ABC): + api_key_env: tuple[str, ...] = () + api_base_env: tuple[str, ...] = () + + def get_default_api_base(self) -> str | None: + return None + + def missing_api_base_message(self, custom_llm_provider: str) -> str: + return f"api_base is required for Extraction provider '{custom_llm_provider}'" + + def resolve_api_base(self, api_base: str | None) -> str | None: + return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base() + + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key or self._first_secret(self.api_key_env) + + @staticmethod + def _first_secret(names: tuple[str, ...]) -> str | None: + return next((value for value in (get_secret_str(name) for name in names) if value), None) + + def canonical_model(self, model: str) -> str: + return model + + def request_model(self, model: str) -> str: + return model + + def validate_environment( + self, + headers: Mapping[str, str], + model: str, + api_key: str | None, + litellm_params: Mapping[str, object], + ) -> Mapping[str, str]: + return { + **{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS}, + **({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}), + "Content-Type": "application/json", + } + + @abstractmethod + def get_complete_url(self, api_base: str, model: str) -> str: ... + + def transform_extraction_request(self, model: str, request: ExtractionRequestBody) -> Mapping[str, object]: + return {**request.model_dump(mode="json", exclude_unset=True), "model": self.request_model(model)} + + def unwrap_response(self, payload: object) -> object: + return payload + + def transform_extraction_response( + self, + model: str, + custom_llm_provider: str, + raw_response: httpx.Response, + ) -> ExtractionResponse: + payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content) + try: + return _RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload)) + except ValidationError as error: + raise BaseLLMException( + status_code=500, + message=f"Extraction provider '{custom_llm_provider}' returned an unexpected response: {error}", + ) from error + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: Mapping[str, str] | httpx.Headers, + ) -> BaseLLMException: + return BaseLLMException(status_code=status_code, message=error_message, headers=httpx.Headers(headers)) diff --git a/litellm/llms/cloudflare/common_utils.py b/litellm/llms/cloudflare/common_utils.py new file mode 100644 index 00000000000..9f1293cb9c4 --- /dev/null +++ b/litellm/llms/cloudflare/common_utils.py @@ -0,0 +1,53 @@ +from collections.abc import Mapping +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm.secret_managers.main import ( + get_secret_str, + normalize_nonempty_secret_str, +) + +CLOUDFLARE_MISSING_API_BASE_MESSAGE: Final = ( + "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly" +) +_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def workers_ai_default_api_base() -> str | None: + account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) + if account_id is None: + return None + return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run" + + +def workers_ai_canonical_model(model: str) -> str: + if model.startswith("@cf/"): + return model + return f"@cf/cloudflare/{model}" + + +def workers_ai_request_model(model: str) -> str: + return model.rsplit("/", maxsplit=1)[-1] + + +def workers_ai_run_url(api_base: str, model: str) -> str: + normalized_api_base: Final = api_base.rstrip("/") + if normalized_api_base.endswith("/ai/v1"): + return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" + if normalized_api_base.endswith("/ai/run"): + return f"{normalized_api_base}/{model}" + return f"{normalized_api_base}/ai/run/{model}" + + +def unwrap_workers_ai_result(payload: object) -> object: + try: + response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) + except ValidationError: + return payload + if "answers" in response_mapping: + return response_mapping + try: + return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result")) + except ValidationError: + return response_mapping diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 967935edfb4..7c776367886 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -1,16 +1,13 @@ -from collections.abc import Mapping -from typing import Final - -from pydantic import TypeAdapter, ValidationError - from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.secret_managers.main import ( - get_secret_str, - normalize_nonempty_secret_str, +from litellm.llms.cloudflare.common_utils import ( + CLOUDFLARE_MISSING_API_BASE_MESSAGE, + unwrap_workers_ai_result, + workers_ai_canonical_model, + workers_ai_default_api_base, + workers_ai_request_model, + workers_ai_run_url, ) -_RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) - class CloudflareDecisionsConfig(BaseDecisionsConfig): api_key_env = ("CLOUDFLARE_API_KEY",) @@ -23,38 +20,19 @@ class CloudflareDecisionsConfig(BaseDecisionsConfig): inlines_remote_images = True def get_default_api_base(self) -> str | None: - account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) - if account_id is None: - return None - return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run" + return workers_ai_default_api_base() def missing_api_base_message(self, custom_llm_provider: str) -> str: - return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly" + return CLOUDFLARE_MISSING_API_BASE_MESSAGE def canonical_model(self, model: str) -> str: - if model.startswith("@cf/"): - return model - return f"@cf/cloudflare/{model}" + return workers_ai_canonical_model(model) def request_model(self, model: str) -> str: - return model.rsplit("/", maxsplit=1)[-1] + return workers_ai_request_model(model) def get_complete_url(self, api_base: str, model: str) -> str: - normalized_api_base: Final = api_base.rstrip("/") - if normalized_api_base.endswith("/ai/v1"): - return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" - if normalized_api_base.endswith("/ai/run"): - return f"{normalized_api_base}/{model}" - return f"{normalized_api_base}/ai/run/{model}" + return workers_ai_run_url(api_base, model) def unwrap_response(self, payload: object) -> object: - try: - response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) - except ValidationError: - return payload - if "answers" in response_mapping: - return response_mapping - try: - return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result")) - except ValidationError: - return response_mapping + return unwrap_workers_ai_result(payload) diff --git a/litellm/llms/cloudflare/extraction/guardrail_translation/__init__.py b/litellm/llms/cloudflare/extraction/guardrail_translation/__init__.py new file mode 100644 index 00000000000..3fda941854a --- /dev/null +++ b/litellm/llms/cloudflare/extraction/guardrail_translation/__init__.py @@ -0,0 +1,11 @@ +from typing import Final + +from litellm.llms.cloudflare.extraction.guardrail_translation.handler import ExtractionHandler +from litellm.types.utils import CallTypes + +guardrail_translation_mappings: Final = { + CallTypes.extraction: ExtractionHandler, + CallTypes.aextraction: ExtractionHandler, +} + +__all__ = ["ExtractionHandler", "guardrail_translation_mappings"] diff --git a/litellm/llms/cloudflare/extraction/guardrail_translation/handler.py b/litellm/llms/cloudflare/extraction/guardrail_translation/handler.py new file mode 100644 index 00000000000..607d6fac48b --- /dev/null +++ b/litellm/llms/cloudflare/extraction/guardrail_translation/handler.py @@ -0,0 +1,276 @@ +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain, groupby +from types import MappingProxyType +from typing import TYPE_CHECKING, Annotated, Final, Literal, TypeAlias, TypeVar + +from pydantic import Field, JsonValue, TypeAdapter + +from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs +from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation +from litellm.types.extraction import ( + EXTRACTION_STATE_REWRITTEN_METADATA_KEY, + ExtractionAnswer, + ExtractionBase64Image, + ExtractionResponse, +) +from litellm.types.utils import GenericGuardrailAPIInputs + +if TYPE_CHECKING: + from litellm.integrations.custom_guardrail import CustomGuardrail + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth + +_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, Mapping[str, object]]]] = TypeAdapter( + Mapping[str, Mapping[str, object]] +) +_METADATA_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) +_IMAGES_ADAPTER: Final[TypeAdapter[tuple[ExtractionBase64Image | JsonValue, ...]]] = TypeAdapter( + tuple[Annotated[ExtractionBase64Image | JsonValue, Field(union_mode="left_to_right")], ...] +) + + +_AnswerKey: TypeAlias = Literal["value", "guess", "mean", "median", "p10", "p90", "quantiles"] +_ESTIMATE_KEYS: Final[tuple[_AnswerKey, ...]] = ("value", "guess", "mean", "median", "p10", "p90", "quantiles") +_STATISTIC_KEYS: Final[frozenset[_AnswerKey]] = frozenset(("mean", "median", "p10", "p90", "quantiles")) +_NULLED_ANSWER: Final[Mapping[str, None]] = MappingProxyType( + dict.fromkeys(("value", "guess", "mean", "median", "p10", "p90", "quantiles", "steps", "span", "spans"), None) +) + + +@dataclass(frozen=True, slots=True) +class _AnswerText: + name: str + key: _AnswerKey + item: int | None + text: str + + +_SPEC_TEXT_KEYS: Final = ("instructions", "unit") + + +@dataclass(frozen=True, slots=True) +class _SpecText: + name: str + key: str + text: str + + +def _spec_texts_of(name: str, spec: Mapping[str, object]) -> tuple[_SpecText, ...]: + return tuple( + _SpecText(name=name, key=key, text=text) for key in _SPEC_TEXT_KEYS if isinstance(text := spec.get(key), str) + ) + + +def _spec_texts(fields: Mapping[str, Mapping[str, object]]) -> tuple[_SpecText, ...]: + return tuple(chain.from_iterable(_spec_texts_of(name, spec) for name, spec in fields.items())) + + +_NamedText = TypeVar("_NamedText", _AnswerText, _SpecText) + + +def _scans_by_name(scanned: Iterable[tuple[_NamedText, str]]) -> Mapping[str, tuple[tuple[_NamedText, str], ...]]: + return {name: tuple(group) for name, group in groupby(scanned, key=lambda pair: pair[0].name)} + + +def _guardrailed_spec(spec: Mapping[str, object], scanned: Sequence[tuple[_SpecText, str]]) -> Mapping[str, object]: + return {**spec, **{text.key: guardrailed for text, guardrailed in scanned}} + + +def _metadata_key(data: Mapping[str, object]) -> Literal["metadata", "litellm_metadata"]: + return get_metadata_variable_name_from_kwargs(data) + + +def _metadata(data: Mapping[str, object]) -> Mapping[str, object]: + return _METADATA_ADAPTER.validate_python(data.get(_metadata_key(data)) or {}) + + +def _is_text_answer(answer: ExtractionAnswer, spec: Mapping[str, object] | None) -> bool: + return answer.type == "text" or (answer.type == "list" and (spec is None or spec.get("items") == "text")) + + +def _image_urls(images: object) -> tuple[str, ...]: + if not isinstance(images, list): + return () + return tuple( + f"data:{image.content_type};base64,{image.base64}" if isinstance(image, ExtractionBase64Image) else image + for image in _IMAGES_ADAPTER.validate_python(images) + if isinstance(image, (str, ExtractionBase64Image)) + ) + + +def _scalar_text(value: JsonValue, whole: bool) -> str | None: + if isinstance(value, bool): + return None + if isinstance(value, float) and whole: + return format(value, ".0f") + return str(value) if isinstance(value, (str, int, float)) else None + + +def _item_texts(items: Sequence[JsonValue], whole: bool) -> tuple[tuple[int, str], ...]: + return tuple((index, text) for index, item in enumerate(items) if (text := _scalar_text(item, whole)) is not None) + + +def _value_texts(name: str, key: _AnswerKey, value: JsonValue) -> tuple[_AnswerText, ...]: + whole: Final = key in _STATISTIC_KEYS + if isinstance(value, list): + return tuple( + _AnswerText(name=name, key=key, item=index, text=text) for index, text in _item_texts(value, whole) + ) + text: Final = _scalar_text(value, whole) + return () if text is None else (_AnswerText(name=name, key=key, item=None, text=text),) + + +def _step_texts(name: str, step: JsonValue) -> tuple[_AnswerText, ...]: + if not isinstance(step, dict): + return () + return tuple(chain.from_iterable(_value_texts(name, key, step.get(key)) for key in _ESTIMATE_KEYS)) + + +def _answer_texts(name: str, answer: ExtractionAnswer) -> tuple[_AnswerText, ...]: + return ( + *_value_texts(name, "value", answer.value), + *_value_texts(name, "guess", answer.guess), + *_value_texts(name, "mean", answer.mean), + *_value_texts(name, "median", answer.median), + *_value_texts(name, "p10", answer.p10), + *_value_texts(name, "p90", answer.p90), + *_value_texts(name, "quantiles", list(answer.quantiles or ())), + *chain.from_iterable(_step_texts(name, step) for step in answer.steps or ()), + ) + + +def _guardrailed_value(value: JsonValue, scanned: Mapping[int | None, str]) -> JsonValue: + if isinstance(value, list): + return [scanned.get(index, item) for index, item in enumerate(value)] + return scanned.get(None, value) + + +def _scans_of(key: _AnswerKey, scanned: Sequence[tuple[_AnswerText, str]]) -> Mapping[int | None, str]: + return {text.item: guardrailed for text, guardrailed in scanned if text.key == key} + + +def _guardrailed_answer( + answer: ExtractionAnswer, + scanned: Sequence[tuple[_AnswerText, str]], + drop_spans: bool, + is_text: bool, +) -> ExtractionAnswer: + if not is_text and any(text.text != guardrailed for text, guardrailed in scanned): + return answer.model_copy(update=_NULLED_ANSWER) + value_scans: Final = _scans_of("value", scanned if is_text else ()) + guess_scans: Final = _scans_of("guess", scanned if is_text else ()) + update: Final = { + **({"value": _guardrailed_value(answer.value, value_scans)} if value_scans else {}), + **({"guess": _guardrailed_value(answer.guess, guess_scans)} if guess_scans else {}), + **({"span": None, "spans": None} if drop_spans else {}), + } + return answer.model_copy(update=update) if update else answer + + +async def _scanned_texts( + texts: Sequence[str], + model: object, + guardrail_to_apply: "CustomGuardrail", + request_data: dict[str, object], # mutable-ok: CustomGuardrail.apply_guardrail takes the request as a plain dict + input_type: Literal["request", "response"], + litellm_logging_obj: "LiteLLMLoggingObj | None", + images: Sequence[str] = (), +) -> tuple[str, ...]: + unique_texts: Final = tuple(dict.fromkeys(texts)) + inputs: Final = GenericGuardrailAPIInputs(texts=list(unique_texts), model=model if isinstance(model, str) else None) + if images: + inputs["images"] = list(images) + guardrailed: Final = await guardrail_to_apply.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # CustomGuardrail.apply_guardrail types request_data as a bare dict + inputs=inputs, + request_data=request_data, + input_type=input_type, + logging_obj=litellm_logging_obj, + ) + returned: Final = guardrailed.get("texts", []) + guardrailed_by_text: Final = MappingProxyType( + {text: returned[index] if index < len(returned) else text for index, text in enumerate(unique_texts)} + ) + return tuple(guardrailed_by_text[text] for text in texts) + + +class ExtractionHandler(BaseTranslation): + async def process_input_messages( + self, + data: dict[str, object], # mutable-ok: BaseTranslation hands over the proxy's request dict + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: "LiteLLMLoggingObj | None" = None, + ) -> Mapping[str, object]: + state: Final = data.get("state") + states: Final = (state,) if isinstance(state, str) and state else () + fields: Final = _FIELDS_ADAPTER.validate_python(data.get("fields") or {}) + spec_texts: Final = _spec_texts(fields) + images: Final = _image_urls(data.get("images")) + if not states and not spec_texts and not images: + return data + scanned: Final = await _scanned_texts( + (*states, *(text.text for text in spec_texts)), + data.get("model"), + guardrail_to_apply, + data, + "request", + litellm_logging_obj, + images, + ) + spec_scans: Final = _scans_by_name(zip(spec_texts, scanned[len(states) :])) + state_rewritten: Final = bool(states) and scanned[0] != states[0] + return { + **data, + **({"state": scanned[0]} if states else {}), + **( + {_metadata_key(data): {**_metadata(data), EXTRACTION_STATE_REWRITTEN_METADATA_KEY: True}} + if state_rewritten + else {} + ), + "fields": {name: _guardrailed_spec(spec, spec_scans.get(name, ())) for name, spec in fields.items()}, + } + + async def process_output_response( + self, + response: object, + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: "LiteLLMLoggingObj | None" = None, + user_api_key_dict: "UserAPIKeyAuth | None" = None, + request_data: dict[str, object] | None = None, # mutable-ok: BaseTranslation hands over the request dict + ) -> object: + if not isinstance(response, ExtractionResponse): + return response + texts: Final = tuple( + chain.from_iterable(_answer_texts(name, answer) for name, answer in response.answers.items()) + ) + state_texts: Final = (response.state_text,) if response.state_text else () + request: Final = request_data or {} + scanned: Final = ( + await _scanned_texts( + (*(text.text for text in texts), *state_texts), + response.model, + guardrail_to_apply, + request, + "response", + litellm_logging_obj, + ) + if texts or state_texts + else () + ) + answer_scans: Final = _scans_by_name(zip(texts, scanned)) + state_text: Final = next(iter(scanned[len(texts) :]), response.state_text) + specs: Final = _FIELDS_ADAPTER.validate_python(request.get("fields") or {}) + return response.model_copy( + update={ + "answers": { + name: _guardrailed_answer( + answer, + answer_scans.get(name, ()), + drop_spans=state_text != response.state_text, + is_text=_is_text_answer(answer, specs.get(name)), + ) + for name, answer in response.answers.items() + }, + **({"state_text": state_text} if state_texts else {}), + } + ) diff --git a/litellm/llms/cloudflare/extraction/transformation.py b/litellm/llms/cloudflare/extraction/transformation.py new file mode 100644 index 00000000000..7aa3bf6316c --- /dev/null +++ b/litellm/llms/cloudflare/extraction/transformation.py @@ -0,0 +1,32 @@ +from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig +from litellm.llms.cloudflare.common_utils import ( + CLOUDFLARE_MISSING_API_BASE_MESSAGE, + unwrap_workers_ai_result, + workers_ai_canonical_model, + workers_ai_default_api_base, + workers_ai_request_model, + workers_ai_run_url, +) + + +class CloudflareExtractionConfig(BaseExtractionConfig): + api_key_env = ("CLOUDFLARE_API_KEY",) + api_base_env = ("CLOUDFLARE_API_BASE",) + + def get_default_api_base(self) -> str | None: + return workers_ai_default_api_base() + + def missing_api_base_message(self, custom_llm_provider: str) -> str: + return CLOUDFLARE_MISSING_API_BASE_MESSAGE + + def canonical_model(self, model: str) -> str: + return workers_ai_canonical_model(model) + + def request_model(self, model: str) -> str: + return workers_ai_request_model(model) + + def get_complete_url(self, api_base: str, model: str) -> str: + return workers_ai_run_url(api_base, model) + + def unwrap_response(self, payload: object) -> object: + return unwrap_workers_ai_result(payload) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b1e55375f16..ac2ac1fe31b 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -76,6 +76,7 @@ from litellm.llms.base_llm.containers.transformation import BaseContainerConfig from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig +from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, BaseFileUploadStream, @@ -128,6 +129,7 @@ from litellm.types.containers.main import ( DeleteContainerResult, ) from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse +from litellm.types.extraction import ExtractionResponse from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, @@ -1700,6 +1702,115 @@ class BaseLLMHTTPHandler: model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request ) + def _prepare_extraction_request( + self, + model: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseExtractionConfig, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + litellm_params: Mapping[str, object], + ) -> tuple[str, Mapping[str, str]]: + outbound_headers: Final = provider_config.validate_environment( + headers=headers, + model=model, + api_key=api_key, + litellm_params=litellm_params, + ) + url: Final = provider_config.get_complete_url(api_base=api_base, model=model) + if logging_obj is not None: + logging_obj.pre_call( # pyright: ignore[reportUnknownMemberType] # Logging.pre_call is untyped + input=dict(body), + api_key=api_key, + model=model, + additional_args={"api_base": url, "complete_input_dict": dict(body), "headers": outbound_headers}, + ) + return url, outbound_headers + + def extraction( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseExtractionConfig, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], + client: HTTPHandler | None = None, + ) -> ExtractionResponse: + url, outbound_headers = self._prepare_extraction_request( + model=model, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + api_base=api_base, + api_key=api_key, + headers=headers, + litellm_params=litellm_params, + ) + sync_httpx_client: Final = client if client is not None else get_httpx_client() + try: + response: Final = sync_httpx_client.post( # pyright: ignore[reportUnknownMemberType] # untyped + url, + json=dict(body), + headers=dict(outbound_headers), + timeout=timeout, + logging_obj=logging_obj, + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_extraction_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response + ) + + async def aextraction( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseExtractionConfig, + body: Mapping[str, object], + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + litellm_params: Mapping[str, object], + client: AsyncHTTPHandler | None = None, + ) -> ExtractionResponse: + url, outbound_headers = self._prepare_extraction_request( + model=model, + logging_obj=logging_obj, + provider_config=provider_config, + body=body, + api_base=api_base, + api_key=api_key, + headers=headers, + litellm_params=litellm_params, + ) + async_httpx_client: Final = ( + client + if client is not None + else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider)) + ) + try: + response: Final = await async_httpx_client.post( # pyright: ignore[reportUnknownMemberType] # untyped + url, + json=dict(body), + headers=dict(outbound_headers), + timeout=timeout, + logging_obj=logging_obj, + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_extraction_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response + ) + def _prepare_audio_transcription_request( self, model: str, @@ -6462,6 +6573,7 @@ class BaseLLMHTTPHandler: provider_config: Union[ BaseConfig, BaseDecisionsConfig, + BaseExtractionConfig, BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index c6aea8ec98a..04a3f9225ce 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -16580,6 +16580,24 @@ "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", "supports_vision": true }, + "cloudflare/clairic": { + "input_cost_per_token": 1e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "extraction", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/clairic/", + "supports_vision": true + }, + "cloudflare/@cf/cloudflare/clairic": { + "input_cost_per_token": 1e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "extraction", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/clairic/", + "supports_vision": true + }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, "litellm_provider": "cloudflare", diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index a7adb5c1bf5..0e2eed73364 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -275,7 +275,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( LazyFeature( name="decisions", module_path="litellm.proxy.decisions_endpoints.endpoints", - path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone"), + path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone", "/v1/extraction", "/extraction"), ), LazyFeature( name="claude_code_marketplace", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 7198d74909c..545b317055a 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9829,6 +9829,249 @@ "title": "DecisionsUsage", "type": "object" }, + "ExtractionAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Confidence" + }, + "guess": { + "$ref": "#/components/schemas/JsonValue" + }, + "mean": { + "$ref": "#/components/schemas/JsonValue" + }, + "median": { + "$ref": "#/components/schemas/JsonValue" + }, + "p10": { + "$ref": "#/components/schemas/JsonValue" + }, + "p90": { + "$ref": "#/components/schemas/JsonValue" + }, + "p_above": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "P Above" + }, + "p_below": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "P Below" + }, + "p_null": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "title": "P Null" + }, + "quantiles": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/JsonValue" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Quantiles" + }, + "span": { + "anyOf": [ + { + "items": { + "type": "integer" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Span" + }, + "spans": { + "anyOf": [ + { + "items": { + "items": { + "type": "integer" + }, + "type": "array" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Spans" + }, + "steps": { + "anyOf": [ + { + "items": { + "$ref": "#/components/schemas/JsonValue" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "title": "Steps" + }, + "type": { + "title": "Type", + "type": "string" + }, + "value": { + "$ref": "#/components/schemas/JsonValue", + "description": "The answer, null when the input does not determine it" + } + }, + "required": [ + "type", + "value" + ], + "title": "ExtractionAnswer", + "type": "object" + }, + "ExtractionResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "additionalProperties": { + "$ref": "#/components/schemas/ExtractionAnswer" + }, + "title": "Answers", + "type": "object" + }, + "model": { + "title": "Model", + "type": "string" + }, + "state_text": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "State Text" + }, + "truncated": { + "title": "Truncated", + "type": "boolean" + }, + "usage": { + "$ref": "#/components/schemas/ExtractionUsage" + } + }, + "required": [ + "model", + "answers", + "truncated", + "usage" + ], + "title": "ExtractionResponse", + "type": "object" + }, + "ExtractionUsage": { + "additionalProperties": true, + "properties": { + "image_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Image Tokens" + }, + "input_tokens": { + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "title": "Output Tokens", + "type": "integer" + }, + "passes": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Passes" + }, + "schema_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "Schema Tokens" + }, + "state_tokens": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "title": "State Tokens" + } + }, + "required": [ + "input_tokens", + "output_tokens" + ], + "title": "ExtractionUsage", + "type": "object" + }, + "JsonValue": {}, "NoulAnswer": { "additionalProperties": true, "properties": { @@ -10765,6 +11008,403 @@ ] } }, + "/extraction": { + "post": { + "description": "Read a `state` text and up to 4 images and get one typed answer per named field in `fields`.\n\nField types are `text`, `number`, `date`, `time`, `datetime`, `duration`, `list` (set `items`), `estimate`\n(set `min` and `max`) and `forecast` (set `min`, `max` and `horizon`). Each answer carries its `value`, which is\nnull when the input does not determine it, plus `confidence`, `p_null`, text `span`s into the state and, for\nestimates, `mean`, `median`, `p10` and `p90`. Images are data URLs or `{content_type, base64}` objects, remote\nURLs are not accepted. An unknown field type, a field name that is not 1 to 64 letters, digits, `_`, `.` or `-`\nstarting with a letter or `_`, a remote image URL or more than 4 images is rejected with a 400 before any\nprovider is called. Any other top-level key, `stream: true` and OpenAI generation settings such as `temperature`\nincluded, is rejected with a 400 naming it unless `drop_params` or the deployment's `additional_drop_params`\ndrops it. Settings inside `fields` and `options` go to the provider as sent, so a field missing a setting its\ntype needs or an empty `state` with no images comes back with the provider's own error. `model` is any\nextraction model in the proxy model list, such as Cloudflare Clairic. The request and the answers keep the\nprovider's format.", + "operationId": "extraction_extraction_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "fields": { + "days_to_pay": { + "max": 90, + "min": 0, + "type": "estimate" + }, + "invoice_date": { + "type": "date" + }, + "items": { + "instructions": "What was bought?", + "items": "text", + "type": "list" + }, + "total": { + "type": "number", + "unit": "USD" + }, + "vendor": { + "instructions": "Who sent the invoice?", + "type": "text" + } + }, + "model": "clairic", + "state": "Invoice #4471 from Acme Tools, dated 2026-03-03. 12 hammers. Total $174. Payment due in 30 days." + }, + "schema": { + "additionalProperties": true, + "properties": { + "fields": { + "description": "Field name to typed field spec. Each name becomes a key in the answers", + "minProperties": 1, + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_.\\-]{0,63}$": { + "additionalProperties": true, + "properties": { + "decimals": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Numbers, durations and number lists only, 0 to 6", + "title": "Decimals" + }, + "horizon": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Forecast only, number of steps", + "title": "Horizon" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "What the field should contain", + "title": "Instructions" + }, + "integer": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Numbers and durations only", + "title": "Integer" + }, + "interval": { + "anyOf": [ + { + "enum": [ + "step", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Interval" + }, + "items": { + "anyOf": [ + { + "enum": [ + "text", + "number", + "date" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Item type, required for list fields", + "title": "Items" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Estimate and forecast upper bound", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Estimate and forecast lower bound", + "title": "Min" + }, + "null_threshold": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Probability of null above which the value is null", + "title": "Null Threshold" + }, + "scale": { + "anyOf": [ + { + "enum": [ + "linear", + "log" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Scale" + }, + "step": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Estimate grid step, or month or year", + "title": "Step" + }, + "type": { + "description": "The answer type for this field", + "enum": [ + "text", + "number", + "date", + "time", + "datetime", + "duration", + "list", + "estimate", + "forecast" + ], + "title": "Type", + "type": "string" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Unit label. For durations, seconds through years", + "title": "Unit" + } + }, + "required": [ + "type" + ], + "title": "ExtractionField", + "type": "object" + } + }, + "title": "Fields", + "type": "object" + }, + "images": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "pattern": "^[Dd][Aa][Tt][Aa]:", + "type": "string" + }, + { + "additionalProperties": true, + "properties": { + "base64": { + "description": "Base64-encoded image bytes", + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "The image media type, such as image/png", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "ExtractionBase64Image", + "type": "object" + } + ] + }, + "maxItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Up to 4 images as data URLs or content_type plus base64, no remote URLs", + "title": "Images" + }, + "model": { + "description": "An extraction model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "options": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "include_quantiles": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Add 99 percentiles to estimates", + "title": "Include Quantiles" + }, + "return_state_text": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Return the text that spans index into", + "title": "Return State Text" + } + }, + "title": "ExtractionOptions", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "reference_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "ISO date that anchors dates written without a year", + "title": "Reference Date" + }, + "state": { + "default": "", + "description": "The text to read. Must be non-empty unless images are sent. Text answers are spans of it", + "title": "State", + "type": "string" + } + }, + "required": [ + "fields", + "model" + ], + "title": "ExtractionRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExtractionResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Extract typed fields from a state", + "tags": [ + "decisions" + ] + } + }, "/systemone": { "post": { "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", @@ -11775,6 +12415,403 @@ ] } }, + "/v1/extraction": { + "post": { + "description": "Read a `state` text and up to 4 images and get one typed answer per named field in `fields`.\n\nField types are `text`, `number`, `date`, `time`, `datetime`, `duration`, `list` (set `items`), `estimate`\n(set `min` and `max`) and `forecast` (set `min`, `max` and `horizon`). Each answer carries its `value`, which is\nnull when the input does not determine it, plus `confidence`, `p_null`, text `span`s into the state and, for\nestimates, `mean`, `median`, `p10` and `p90`. Images are data URLs or `{content_type, base64}` objects, remote\nURLs are not accepted. An unknown field type, a field name that is not 1 to 64 letters, digits, `_`, `.` or `-`\nstarting with a letter or `_`, a remote image URL or more than 4 images is rejected with a 400 before any\nprovider is called. Any other top-level key, `stream: true` and OpenAI generation settings such as `temperature`\nincluded, is rejected with a 400 naming it unless `drop_params` or the deployment's `additional_drop_params`\ndrops it. Settings inside `fields` and `options` go to the provider as sent, so a field missing a setting its\ntype needs or an empty `state` with no images comes back with the provider's own error. `model` is any\nextraction model in the proxy model list, such as Cloudflare Clairic. The request and the answers keep the\nprovider's format.", + "operationId": "extraction_v1_extraction_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "fields": { + "days_to_pay": { + "max": 90, + "min": 0, + "type": "estimate" + }, + "invoice_date": { + "type": "date" + }, + "items": { + "instructions": "What was bought?", + "items": "text", + "type": "list" + }, + "total": { + "type": "number", + "unit": "USD" + }, + "vendor": { + "instructions": "Who sent the invoice?", + "type": "text" + } + }, + "model": "clairic", + "state": "Invoice #4471 from Acme Tools, dated 2026-03-03. 12 hammers. Total $174. Payment due in 30 days." + }, + "schema": { + "additionalProperties": true, + "properties": { + "fields": { + "description": "Field name to typed field spec. Each name becomes a key in the answers", + "minProperties": 1, + "patternProperties": { + "^[A-Za-z_][A-Za-z0-9_.\\-]{0,63}$": { + "additionalProperties": true, + "properties": { + "decimals": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Numbers, durations and number lists only, 0 to 6", + "title": "Decimals" + }, + "horizon": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Forecast only, number of steps", + "title": "Horizon" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "What the field should contain", + "title": "Instructions" + }, + "integer": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Numbers and durations only", + "title": "Integer" + }, + "interval": { + "anyOf": [ + { + "enum": [ + "step", + "minute", + "hour", + "day", + "week", + "month", + "quarter", + "year" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Interval" + }, + "items": { + "anyOf": [ + { + "enum": [ + "text", + "number", + "date" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Item type, required for list fields", + "title": "Items" + }, + "max": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Estimate and forecast upper bound", + "title": "Max" + }, + "min": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Estimate and forecast lower bound", + "title": "Min" + }, + "null_threshold": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "description": "Probability of null above which the value is null", + "title": "Null Threshold" + }, + "scale": { + "anyOf": [ + { + "enum": [ + "linear", + "log" + ], + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Scale" + }, + "step": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Estimate grid step, or month or year", + "title": "Step" + }, + "type": { + "description": "The answer type for this field", + "enum": [ + "text", + "number", + "date", + "time", + "datetime", + "duration", + "list", + "estimate", + "forecast" + ], + "title": "Type", + "type": "string" + }, + "unit": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Unit label. For durations, seconds through years", + "title": "Unit" + } + }, + "required": [ + "type" + ], + "title": "ExtractionField", + "type": "object" + } + }, + "title": "Fields", + "type": "object" + }, + "images": { + "anyOf": [ + { + "items": { + "anyOf": [ + { + "pattern": "^[Dd][Aa][Tt][Aa]:", + "type": "string" + }, + { + "additionalProperties": true, + "properties": { + "base64": { + "description": "Base64-encoded image bytes", + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "The image media type, such as image/png", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "ExtractionBase64Image", + "type": "object" + } + ] + }, + "maxItems": 4, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "Up to 4 images as data URLs or content_type plus base64, no remote URLs", + "title": "Images" + }, + "model": { + "description": "An extraction model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "options": { + "anyOf": [ + { + "additionalProperties": true, + "properties": { + "include_quantiles": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Add 99 percentiles to estimates", + "title": "Include Quantiles" + }, + "return_state_text": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "description": "Return the text that spans index into", + "title": "Return State Text" + } + }, + "title": "ExtractionOptions", + "type": "object" + }, + { + "type": "null" + } + ] + }, + "reference_date": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "ISO date that anchors dates written without a year", + "title": "Reference Date" + }, + "state": { + "default": "", + "description": "The text to read. Must be non-empty unless images are sent. Text answers are spans of it", + "title": "State", + "type": "string" + } + }, + "required": [ + "fields", + "model" + ], + "title": "ExtractionRequest", + "type": "object" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ExtractionResponse" + } + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Extract typed fields from a state", + "tags": [ + "decisions" + ] + } + }, "/v1/systemone": { "post": { "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 315533d6d0f..97a2fc31fb0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -377,6 +377,8 @@ class LiteLLMRoutes(enum.Enum): "/v1/decisions", "/systemone", "/v1/systemone", + "/extraction", + "/v1/extraction", # OCR "/ocr", "/v1/ocr", diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index cbbd1548301..cdbbd6029fa 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -32,6 +32,7 @@ _MANAGED_MODEL_ROUTES: Final = frozenset( "rerank", "decisions", "systemone", + "extraction", "ocr", ), ) @@ -77,6 +78,7 @@ _MODEL_ROUTE_KINDS: Final[ "/rerank": "body", "/decisions": "body", "/systemone": "body", + "/extraction": "body", "/messages/count_tokens": "body", ":countTokens": "path", } diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 2ffdd02cdf5..59db23820fc 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -133,12 +133,14 @@ from litellm.router import Router from litellm.router_utils.add_retry_fallback_headers import get_hidden_params_dict from litellm.router_utils.auto_router_model_naming import AUTO_ROUTER_MODEL_PREFIX from litellm.router_utils.common_utils import resolve_model_group_alias, resolve_served_model +from litellm.types.extraction import ExtractionResponse from litellm.types.guardrails import GuardrailEventHooks from litellm.types.router import RouterRateLimitError from litellm.types.router_weights import validate_router_weights _LateResponseT = TypeVar("_LateResponseT", bound=Response) _LlmCallT = TypeVar("_LlmCallT") +_ResponseT: Final = TypeVar("_ResponseT") _ChunkT: Final = TypeVar("_ChunkT") KNOWN_PROXY_ROUTES: Final = frozenset( @@ -190,6 +192,7 @@ ProxyRouteType: TypeAlias = Literal[ "asearch", "adecisions", "asystemone", + "aextraction", "avideo_generation", "avideo_list", "avideo_status", @@ -1481,11 +1484,11 @@ _is_azure_model_router_request: Final = is_azure_model_router_request def _override_openai_response_model( *, - response_obj: object, + response_obj: _ResponseT, requested_model: str, log_context: str, return_raw_model_name: bool = False, -) -> None: +) -> _ResponseT: """ Force the OpenAI-compatible `model` field in the response to match what the client requested. @@ -1509,7 +1512,7 @@ def _override_openai_response_model( model group name instead of the comma-separated list the client sent. """ if return_raw_model_name or not requested_model: - return + return response_obj hidden_params: Final = get_hidden_params_dict(response_obj) if isinstance(hidden_params, dict): @@ -1522,7 +1525,7 @@ def _override_openai_response_model( log_context, attempted_fallbacks, ) - return + return response_obj # For fastest_response batch completions, use the winning model's group # name rather than the comma-separated list the client sent. @@ -1541,7 +1544,7 @@ def _override_openai_response_model( "%s: fastest_response detected but no model group header found, preserving actual model from response.", log_context, ) - return + return response_obj # Check if this is an Azure Model Router request - if so, preserve the actual model used if is_azure_model_router_request(requested_model, hidden_params): @@ -1549,11 +1552,12 @@ def _override_openai_response_model( "%s: Azure Model Router detected - preserving actual model used from response instead of overriding to router model.", log_context, ) - return + return response_obj + unchanged_response: Final = response_obj if isinstance(response_obj, dict): if "model" not in response_obj: - return + return unchanged_response downstream_model = response_obj.get("model") if downstream_model != requested_model: verbose_proxy_logger.debug( @@ -1563,10 +1567,10 @@ def _override_openai_response_model( downstream_model, ) response_obj["model"] = requested_model - return + return unchanged_response if not hasattr(response_obj, "model"): - return + return response_obj downstream_model = getattr(response_obj, "model", None) if downstream_model != requested_model: @@ -1577,6 +1581,9 @@ def _override_openai_response_model( downstream_model, ) + if isinstance(response_obj, ExtractionResponse): + return response_obj.model_copy(update={"model": requested_model}) + try: setattr(response_obj, "model", requested_model) except Exception as e: @@ -1588,6 +1595,7 @@ def _override_openai_response_model( str(e), exc_info=True, ) + return response_obj _METADATA_BUCKET_KEYS: Final = ("metadata", "litellm_metadata") @@ -2200,6 +2208,7 @@ class ProxyBaseLLMRequestProcessing: "asearch", "adecisions", "asystemone", + "aextraction", "avideo_generation", "avideo_list", "avideo_status", @@ -3226,7 +3235,7 @@ class ProxyBaseLLMRequestProcessing: # Always return the client-requested model name (not provider-prefixed internal identifiers) # for OpenAI-compatible responses. if requested_model_from_client: - _override_openai_response_model( + response = _override_openai_response_model( # rebind-ok: a frozen response comes back as a copy response_obj=response, requested_model=requested_model_from_client, log_context=f"litellm_call_id={logging_obj.litellm_call_id}", diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 1477e9f9591..366842d23a6 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -1,6 +1,6 @@ from collections.abc import Mapping from types import MappingProxyType -from typing import Annotated, Final, Literal +from typing import Annotated, Final, Literal, TypeAlias from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract @@ -18,6 +18,7 @@ from litellm.types.decisions import ( OpenAIDecisionRequestBody, OpenAIDecisionResponse, ) +from litellm.types.extraction import ExtractionRequest, ExtractionRequestBody, ExtractionResponse router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) @@ -25,6 +26,7 @@ _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = Type _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter( OpenAIDecisionRequestBody ) +_EXTRACTION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[ExtractionRequestBody]] = TypeAdapter(ExtractionRequestBody) _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) @@ -73,16 +75,35 @@ _DECISIONS_REQUEST_BODY: Final = inline_request_body( ], }, ) +_EXTRACTION_REQUEST_BODY: Final = inline_request_body( + ExtractionRequest, + { + "model": "clairic", + "state": "Invoice #4471 from Acme Tools, dated 2026-03-03. 12 hammers. Total $174. Payment due in 30 days.", + "fields": { + "vendor": {"type": "text", "instructions": "Who sent the invoice?"}, + "invoice_date": {"type": "date"}, + "total": {"type": "number", "unit": "USD"}, + "items": {"type": "list", "items": "text", "instructions": "What was bought?"}, + "days_to_pay": {"type": "estimate", "min": 0, "max": 90}, + }, + }, +) +_ApiName: TypeAlias = Literal["Decisions", "Extraction"] +_RouteType: TypeAlias = Literal["adecisions", "asystemone", "aextraction"] async def _invalid_request( - raw_data: Mapping[str, object], error: ValidationError, user_api_key_dict: UserAPIKeyAuth + raw_data: Mapping[str, object], + error: ValidationError, + user_api_key_dict: UserAPIKeyAuth, + api_name: _ApiName, ) -> Exception: from litellm.proxy.proxy_server import proxy_logging_obj, version return await ProxyBaseLLMRequestProcessing(data=dict(raw_data)).handle_llm_api_exception( e=BadRequestError( - message=f"Invalid Decisions request: {error}", + message=f"Invalid {api_name} request: {error}", model=str(raw_data.get("model", "")), llm_provider="", ), @@ -95,19 +116,21 @@ async def _invalid_request( _FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "images", "assets", "audio", "videos"}) -def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]: +def _fields_checked_before_routing(data: Mapping[str, object], route_type: _RouteType) -> Mapping[str, object]: + if route_type == "aextraction": + return data return MappingProxyType({key: value for key, value in data.items() if key not in _FIELDS_CHECKED_AFTER_ROUTING}) _DEPLOYMENT_ONLY_FIELDS: Final = frozenset({"model_info"}) -async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: +async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth, api_name: _ApiName) -> dict[str, object]: body: Final = await request.body() try: data: Final = _REQUEST_DATA_ADAPTER.validate_json(body) except ValidationError as error: - raise await _invalid_request(raw_data={}, error=error, user_api_key_dict=user_api_key_dict) + raise await _invalid_request(raw_data={}, error=error, user_api_key_dict=user_api_key_dict, api_name=api_name) return {key: value for key, value in data.items() if key not in _DEPLOYMENT_ONLY_FIELDS} @@ -115,8 +138,10 @@ async def _process_decisions( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, - body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], - route_type: Literal["adecisions", "asystemone"], + body_adapter: ( + TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody] | TypeAdapter[ExtractionRequestBody] + ), + route_type: _RouteType, ) -> object: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, @@ -139,15 +164,17 @@ async def _process_decisions( user_temperature as proxy_user_temperature, ) - data: Final = await _request_data(request, user_api_key_dict) + api_name: Final[_ApiName] = "Extraction" if route_type == "aextraction" else "Decisions" + data: Final = await _request_data(request, user_api_key_dict, api_name) try: - body_adapter.validate_python(_fields_checked_before_routing(data)) + body_adapter.validate_python(_fields_checked_before_routing(data, route_type)) except ValidationError as error: - raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) + raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict, api_name=api_name) general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base) user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model) user_temperature: Final = _OPTIONAL_FLOAT_ADAPTER.validate_python(proxy_user_temperature) + takes_generation_settings: Final = route_type != "aextraction" processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: return await processor.base_process_llm_request( @@ -162,9 +189,9 @@ async def _process_decisions( select_data_generator=None, model=None, user_model=user_model, - user_temperature=user_temperature, + user_temperature=user_temperature if takes_generation_settings else None, user_request_timeout=user_request_timeout, - user_max_tokens=user_max_tokens, + user_max_tokens=user_max_tokens if takes_generation_settings else None, user_api_base=user_api_base, version=version, ) @@ -263,3 +290,53 @@ async def decisions( body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER, route_type="adecisions", ) + + +@router.post( + "/v1/extraction", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["extraction"], + summary="Extract typed fields from a state", + responses={200: {"model": ExtractionResponse}}, + response_model=None, + openapi_extra={"requestBody": _EXTRACTION_REQUEST_BODY}, +) +@router.post( + "/extraction", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["extraction"], + summary="Extract typed fields from a state", + responses={200: {"model": ExtractionResponse}}, + response_model=None, + openapi_extra={"requestBody": _EXTRACTION_REQUEST_BODY}, +) +async def extraction( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> object: + """ + Read a `state` text and up to 4 images and get one typed answer per named field in `fields`. + + Field types are `text`, `number`, `date`, `time`, `datetime`, `duration`, `list` (set `items`), `estimate` + (set `min` and `max`) and `forecast` (set `min`, `max` and `horizon`). Each answer carries its `value`, which is + null when the input does not determine it, plus `confidence`, `p_null`, text `span`s into the state and, for + estimates, `mean`, `median`, `p10` and `p90`. Images are data URLs or `{content_type, base64}` objects, remote + URLs are not accepted. An unknown field type, a field name that is not 1 to 64 letters, digits, `_`, `.` or `-` + starting with a letter or `_`, a remote image URL or more than 4 images is rejected with a 400 before any + provider is called. Any other top-level key, `stream: true` and OpenAI generation settings such as `temperature` + included, is rejected with a 400 naming it unless `drop_params` or the deployment's `additional_drop_params` + drops it. Settings inside `fields` and `options` go to the provider as sent, so a field missing a setting its + type needs or an empty `state` with no images comes back with the provider's own error. `model` is any + extraction model in the proxy model list, such as Cloudflare Clairic. The request and the answers keep the + provider's format. + """ + return await _process_decisions( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + body_adapter=_EXTRACTION_REQUEST_BODY_ADAPTER, + route_type="aextraction", + ) diff --git a/litellm/proxy/health_endpoints/_health_endpoints.py b/litellm/proxy/health_endpoints/_health_endpoints.py index 1e7f67392e1..684f1619512 100644 --- a/litellm/proxy/health_endpoints/_health_endpoints.py +++ b/litellm/proxy/health_endpoints/_health_endpoints.py @@ -2155,6 +2155,7 @@ async def test_model_connection( "ocr", "evaluation", "decisions", + "extraction", ] | None = fastapi.Body( None, diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index d858eff3348..4f2f7f9c256 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -7,6 +7,7 @@ This is currently in development and not yet ready for production. import asyncio import binascii import itertools +import json import logging import os import uuid @@ -42,6 +43,7 @@ from litellm.caching.redis_batch import ( ) from litellm.caching.redis_cache import log_redis_failure from litellm.constants import DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE, INTERNAL_CALL_ORIGIN_METADATA_KEY +from litellm.cost_calculator import get_usage_object from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, @@ -78,6 +80,7 @@ from litellm.router_utils.add_retry_fallback_headers import ( from litellm.router_utils.common_utils import resolve_model_group_alias from litellm.router_utils.ptu_shares import PTUTeamCeiling from litellm.types.caching import RedisPipelineIncrementOperation +from litellm.types.extraction import ExtractionResponse from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType @@ -490,6 +493,7 @@ RESPONSES_API_CALL_TYPES: Final = ("aresponses", "responses") EMBEDDING_API_CALL_TYPES: Final = ("aembedding", "embedding") TEXT_COMPLETION_API_CALL_TYPES: Final = ("atext_completion", "text_completion") RERANK_API_CALL_TYPES: Final = (CallTypes.rerank.value, CallTypes.arerank.value) +EXTRACTION_API_CALL_TYPES: Final = (CallTypes.extraction.value, CallTypes.aextraction.value) GOOGLE_GENAI_NATIVE_CALL_TYPES: Final = ( CallTypes.generate_content.value, CallTypes.agenerate_content.value, @@ -1094,6 +1098,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): or PROXY_MaxParallelRequestsHandler_v3._has_explicit_output_cap(data, call_type) or is_embedding or endpoint_type == EndpointType.DECISIONS + or call_type in EXTRACTION_API_CALL_TYPES ): return effective_cap: Final = max(capped_floor, configured_output_tokens or 0) @@ -1218,6 +1223,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if call_type in RESPONSES_API_CALL_TYPES or call_type in EMBEDDING_API_CALL_TYPES else (None, estimable_data.get("prompt"), None) if call_type in TEXT_COMPLETION_API_CALL_TYPES + else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(estimable_data)), None) + if call_type in EXTRACTION_API_CALL_TYPES else (estimable_data.get("messages"), None, None) if call_type else ( @@ -1245,7 +1252,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): estimated_input_tokens: Final = max(1, total_chars // DEFAULT_CHARS_PER_TOKEN) if total_chars > 0 else 0 explicit_max_tokens: Final = self._get_explicit_output_cap(data, call_type) - is_embedding: Final = self._is_embedding_request(data, call_type) + generates_no_output: Final = ( + self._is_embedding_request(data, call_type) or call_type in EXTRACTION_API_CALL_TYPES + ) base_output_floor: Final = self.no_max_tokens_output_floor(min_configured_tpm_limit) output_floor: Final = ( @@ -1255,7 +1264,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) max_tokens_estimate: Final = ( 0 - if is_embedding or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None) + if generates_no_output + or (explicit_max_tokens is None and total_chars == 0 and configured_output_tokens is None) else explicit_max_tokens if explicit_max_tokens is not None else configured_output_tokens @@ -3834,6 +3844,11 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if isinstance(part, (str, dict)) ) + @staticmethod + def _extraction_input_text(data: Mapping[str, object]) -> str: + state: Final = data.get("state") + return f"{state if isinstance(state, str) else ''}\n{json.dumps(data.get('fields'), default=str)}" + def _estimate_precise_input_tokens(self, data: object, model: str | None, call_type: str | None = None) -> int: """ Model-aware input token estimate for the project ITPM reservation, @@ -3892,6 +3907,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if call_type in RERANK_API_CALL_TYPES else (None, prompt, data.get("tools"), data.get("tool_choice")) if call_type in TEXT_COMPLETION_API_CALL_TYPES + else (None, self._extraction_input_text(_REQUEST_RATE_LIMIT_DATA.validate_python(data)), None, None) + if call_type in EXTRACTION_API_CALL_TYPES else (data.get("messages"), fallback_text, data.get("tools"), data.get("tool_choice")) ) messages, selected_text, countable_tools, countable_tool_choice = selected_inputs @@ -4512,6 +4529,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) -> Usage | ResponseAPIUsage | Mapping[str, object] | None: if isinstance(response_obj, (Usage, ResponseAPIUsage)): return response_obj + if isinstance(response_obj, ExtractionResponse): + return get_usage_object(response_obj) if isinstance( response_obj, (ModelResponse, EmbeddingResponse, TextCompletionResponse, BaseLiteLLMOpenAIResponseObject), @@ -5143,21 +5162,24 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # than parsed out of the body) carry their usage in # ``combined_usage_object`` instead, and would otherwise never charge # the TPM window. - _usage: Usage | dict[str, object] | None = None - if isinstance( - response_obj, - ( - ModelResponse, - EmbeddingResponse, - TextCompletionResponse, - BaseLiteLLMOpenAIResponseObject, - ), - ): - _usage = getattr(response_obj, "usage", None) - else: - _combined_usage: Final = kwargs.get("combined_usage_object") - if isinstance(_combined_usage, Usage): - _usage = _combined_usage + _combined_usage: Final = kwargs.get("combined_usage_object") + _usage: Final[Usage | dict[str, object] | None] = ( + getattr(response_obj, "usage", None) + if isinstance( + response_obj, + ( + ModelResponse, + EmbeddingResponse, + TextCompletionResponse, + BaseLiteLLMOpenAIResponseObject, + ), + ) + else get_usage_object(response_obj) + if isinstance(response_obj, ExtractionResponse) + else _combined_usage + if isinstance(_combined_usage, Usage) + else None + ) total_tokens = self._get_total_tokens_from_usage(usage=_usage, rate_limit_type=rate_limit_type) if total_tokens == 0: total_tokens = self._aggregate_only_total_tokens(usage=_usage) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index e16e14c8d54..25f9f1dff15 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -111,6 +111,7 @@ ROUTE_ENDPOINT_MAPPING: Final = { "asearch": "/search", "adecisions": "/decisions", "asystemone": "/systemone", + "aextraction": "/extraction", "avideo_generation": "/videos", "avideo_list": "/videos", "avideo_status": "/videos/{video_id}", @@ -573,6 +574,7 @@ RouteType = Literal[ "asearch", "adecisions", "asystemone", + "aextraction", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/router.py b/litellm/router.py index 679f4eba1ea..a18ad7c3a97 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2525,6 +2525,11 @@ class Router: self.asystemone = self.factory_function(asystemone, call_type="asystemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable self.systemone = self.factory_function(systemone, call_type="systemone") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable + from litellm.extraction import aextraction, extraction + + self.aextraction = self.factory_function(aextraction, call_type="aextraction") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable + self.extraction = self.factory_function(extraction, call_type="extraction") # pyright: ignore[reportUnknownMemberType] # factory_function takes a bare Callable + def _initialize_video_endpoints(self): """Initialize video endpoints.""" from litellm.videos import ( @@ -7705,6 +7710,8 @@ class Router: "decisions", "asystemone", "systemone", + "aextraction", + "extraction", "aadapter_generate_content", "avideo_generation", "video_generation", @@ -7780,6 +7787,7 @@ class Router: "search", "decisions", "systemone", + "extraction", "video_generation", "video_list", "video_status", @@ -7949,6 +7957,7 @@ class Router: "ocr", "adecisions", "asystemone", + "aextraction", "avideo_generation", "avideo_list", "avideo_status", diff --git a/litellm/types/extraction.py b/litellm/types/extraction.py new file mode 100644 index 00000000000..4311e549b9a --- /dev/null +++ b/litellm/types/extraction.py @@ -0,0 +1,162 @@ +from collections.abc import Mapping, Sequence +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import ( + ConfigDict, + Field, + JsonValue, + PrivateAttr, + StrictBool, + StrictFloat, + StrictInt, + StrictStr, + model_validator, + with_config, +) +from typing_extensions import ReadOnly, Required, Self, TypedDict + +from litellm.types.llms.base import LiteLLMPydanticObjectBase + +ExtractionFieldType: TypeAlias = Literal[ + "text", "number", "date", "time", "datetime", "duration", "list", "estimate", "forecast" +] +ExtractionFieldName: TypeAlias = Annotated[str, Field(pattern=r"^[A-Za-z_][A-Za-z0-9_.\-]{0,63}$")] +JsonNumber: TypeAlias = StrictInt | StrictFloat +ExtractionBound: TypeAlias = JsonNumber | StrictStr +EXTRACTION_STATE_REWRITTEN_METADATA_KEY: Final = "extraction_state_rewritten_by_guardrail" + + +def _is_none(value: object) -> bool: + return value is None + + +_OMITTED_WHEN_NONE: Final = Field(exclude_if=_is_none) +OptionalJson: TypeAlias = Annotated[JsonValue, _OMITTED_WHEN_NONE] +OptionalFloat: TypeAlias = Annotated[float | None, _OMITTED_WHEN_NONE] +OptionalInt: TypeAlias = Annotated[int | None, _OMITTED_WHEN_NONE] + + +class _ClairicObject(LiteLLMPydanticObjectBase): + model_config = ConfigDict(extra="allow", frozen=True) + + +class ExtractionField(_ClairicObject): + type: Annotated[ExtractionFieldType, Field(description="The answer type for this field")] + instructions: Annotated[str | None, Field(description="What the field should contain")] = None + unit: Annotated[str | None, Field(description="Unit label. For durations, seconds through years")] = None + integer: Annotated[StrictBool | None, Field(description="Numbers and durations only")] = None + decimals: Annotated[JsonNumber | None, Field(description="Numbers, durations and number lists only, 0 to 6")] = None + items: Annotated[ + Literal["text", "number", "date"] | None, Field(description="Item type, required for list fields") + ] = None + min: Annotated[ExtractionBound | None, Field(description="Estimate and forecast lower bound")] = None + max: Annotated[ExtractionBound | None, Field(description="Estimate and forecast upper bound")] = None + step: Annotated[ExtractionBound | None, Field(description="Estimate grid step, or month or year")] = None + scale: Literal["linear", "log"] | None = None + null_threshold: Annotated[ + JsonNumber | None, Field(description="Probability of null above which the value is null") + ] = None + horizon: Annotated[JsonNumber | None, Field(description="Forecast only, number of steps")] = None + interval: Literal["step", "minute", "hour", "day", "week", "month", "quarter", "year"] | None = None + + @model_validator(mode="after") + def _list_fields_name_their_item_type(self) -> Self: + if self.type == "list" and self.items is None: + raise ValueError("list fields need items set to text, number or date") + return self + + +class ExtractionBase64Image(_ClairicObject): + content_type: Annotated[str, Field(description="The image media type, such as image/png")] + base64: Annotated[str, Field(description="Base64-encoded image bytes")] + + +ExtractionImage: TypeAlias = Annotated[str, Field(pattern=r"^[Dd][Aa][Tt][Aa]:")] | ExtractionBase64Image + + +class ExtractionOptions(_ClairicObject): + include_quantiles: Annotated[StrictBool | None, Field(description="Add 99 percentiles to estimates")] = None + return_state_text: Annotated[StrictBool | None, Field(description="Return the text that spans index into")] = None + + +class ExtractionRequestBody(_ClairicObject): + state: Annotated[ + str, + Field(description="The text to read. Must be non-empty unless images are sent. Text answers are spans of it"), + ] = "" + fields: Annotated[ + Mapping[ExtractionFieldName, ExtractionField], + Field(min_length=1, description="Field name to typed field spec. Each name becomes a key in the answers"), + ] + images: Annotated[ + tuple[ExtractionImage, ...] | None, + Field(max_length=4, description="Up to 4 images as data URLs or content_type plus base64, no remote URLs"), + ] = None + reference_date: Annotated[str | None, Field(description="ISO date that anchors dates written without a year")] = ( + None + ) + options: ExtractionOptions | None = None + + +class ExtractionRequest(ExtractionRequestBody): + model: Annotated[str, Field(description="An extraction model from the proxy model_list")] + + +class ExtractionAnswer(_ClairicObject): + type: str + value: Annotated[JsonValue, Field(description="The answer, null when the input does not determine it")] + guess: OptionalJson = None + p_null: OptionalFloat = None + confidence: OptionalFloat = None + span: Annotated[tuple[int, ...] | None, _OMITTED_WHEN_NONE] = None + spans: Annotated[tuple[tuple[int, ...], ...] | None, _OMITTED_WHEN_NONE] = None + mean: OptionalJson = None + median: OptionalJson = None + p10: OptionalJson = None + p90: OptionalJson = None + p_below: OptionalFloat = None + p_above: OptionalFloat = None + quantiles: Annotated[tuple[JsonValue, ...] | None, _OMITTED_WHEN_NONE] = None + steps: Annotated[tuple[JsonValue, ...] | None, _OMITTED_WHEN_NONE] = None + + +class ExtractionUsage(_ClairicObject): + input_tokens: int + output_tokens: int + state_tokens: OptionalInt = None + image_tokens: OptionalInt = None + schema_tokens: OptionalInt = None + passes: OptionalInt = None + + +class ExtractionResponse(_ClairicObject): + model: str + answers: Mapping[str, ExtractionAnswer] + truncated: bool + state_text: Annotated[str | None, _OMITTED_WHEN_NONE] = None + usage: ExtractionUsage + + _hidden_params: dict[str, object] = PrivateAttr( # mutable-ok: @client and the proxy write cost into it in place + default_factory=dict + ) + + @property + def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation + return self._hidden_params + + def set_hidden_params(self, params: Mapping[str, object]) -> None: + self._hidden_params.update(params) + + +@with_config(ConfigDict(extra="allow")) +class ExtractionCallParams(TypedDict, total=False): + model: Required[ReadOnly[str]] + state: Required[ReadOnly[str]] + fields: Required[ReadOnly[Mapping[str, Mapping[str, object]]]] + images: ReadOnly[Sequence[str | Mapping[str, object]] | None] + reference_date: ReadOnly[str | None] + api_key: ReadOnly[str | None] + api_base: ReadOnly[str | None] + timeout: ReadOnly[float | None] + custom_llm_provider: ReadOnly[str | None] + extra_headers: ReadOnly[Mapping[str, str] | None] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 61a179b8b65..f86c6cfd4b6 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -458,6 +458,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False): "responses", "decisions", "evaluation", + "extraction", "ocr", "realtime", ] @@ -518,6 +519,8 @@ class CallTypes(str, Enum): adecisions = "adecisions" systemone = "systemone" asystemone = "asystemone" + extraction = "extraction" + aextraction = "aextraction" arealtime = "_arealtime" aresponses_websocket = "_aresponses_websocket" create_batch = "create_batch" @@ -708,6 +711,8 @@ CallTypesLiteral = Literal[ "adecisions", "systemone", "asystemone", + "extraction", + "aextraction", "_arealtime", "_aresponses_websocket", "create_batch", @@ -821,6 +826,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], "/systemone": [CallTypes.asystemone, CallTypes.systemone], "/v1/systemone": [CallTypes.asystemone, CallTypes.systemone], + "/extraction": [CallTypes.aextraction, CallTypes.extraction], + "/v1/extraction": [CallTypes.aextraction, CallTypes.extraction], # Batches "/batches": [CallTypes.acreate_batch, CallTypes.create_batch], "/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch], diff --git a/litellm/utils.py b/litellm/utils.py index a9079f470c7..4687e4fa27a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -390,6 +390,7 @@ if TYPE_CHECKING: from litellm.llms.base_llm.containers.transformation import BaseContainerConfig from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig + from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.llms.base_llm.google_genai.transformation import ( BaseGoogleGenAIGenerateContentConfig, @@ -2166,7 +2167,11 @@ def client(original_function): # Sync logging_obj.stream after deployment hooks (they may convert it). _hook_stream: Final = kwargs.get("stream") - if _hook_stream is not None and logging_obj.stream != _hook_stream: + if ( + _hook_stream is not None + and logging_obj.stream != _hook_stream + and call_type not in _NEVER_STREAMING_CALL_TYPES + ): logging_obj.stream = _hook_stream kwargs["litellm_logging_obj"] = logging_obj @@ -2510,6 +2515,15 @@ _STREAMING_CALL_TYPES: Final = frozenset( } ) +_NEVER_STREAMING_CALL_TYPES: Final = frozenset( + { + CallTypes.extraction, + CallTypes.aextraction, + CallTypes.extraction.value, + CallTypes.aextraction.value, + } +) + def _is_streaming_request( kwargs: dict[str, object], @@ -2521,6 +2535,8 @@ def _is_streaming_request( - if "stream=True" in kwargs (litellm chat completion, litellm text completion, litellm messages) - if call_type is generate_content_stream or agenerate_content_stream (litellm google genai) """ + if call_type in _NEVER_STREAMING_CALL_TYPES: + return False if "stream" in kwargs and kwargs["stream"] is True: return True return call_type in _STREAMING_CALL_TYPES @@ -9539,6 +9555,12 @@ class ProviderConfigManager: return litellm.AzureAIDecisionsConfig() return None + @staticmethod + def get_provider_extraction_config(model: str, provider: LlmProviders) -> BaseExtractionConfig | None: + if provider == LlmProviders.CLOUDFLARE: + return litellm.CloudflareExtractionConfig() + return None + @staticmethod def get_provider_anthropic_messages_config( model: str, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index c6aea8ec98a..04a3f9225ce 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -16580,6 +16580,24 @@ "source": "https://developers.cloudflare.com/workers-ai/platform/pricing/", "supports_vision": true }, + "cloudflare/clairic": { + "input_cost_per_token": 1e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "extraction", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/clairic/", + "supports_vision": true + }, + "cloudflare/@cf/cloudflare/clairic": { + "input_cost_per_token": 1e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 65536, + "mode": "extraction", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/clairic/", + "supports_vision": true + }, "cloudflare/@cf/meta/llama-2-7b-chat-fp16": { "input_cost_per_token": 1.923e-06, "litellm_provider": "cloudflare", diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json index 016d40ea874..d4dfeb25429 100644 --- a/model_prices_and_context_window.schema.json +++ b/model_prices_and_context_window.schema.json @@ -553,6 +553,7 @@ "completion", "embedding", "evaluation", + "extraction", "guardrail", "image_edit", "image_generation", diff --git a/tests/benchmarks/test_extraction_guardrail_benchmarks.py b/tests/benchmarks/test_extraction_guardrail_benchmarks.py new file mode 100644 index 00000000000..0d8a3db42ce --- /dev/null +++ b/tests/benchmarks/test_extraction_guardrail_benchmarks.py @@ -0,0 +1,61 @@ +import asyncio +from typing import Final, Literal + +import pytest + +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.cloudflare.extraction.guardrail_translation.handler import ExtractionHandler +from litellm.types.extraction import ExtractionResponse +from litellm.types.utils import GenericGuardrailAPIInputs + +MANY_FIELDS: Final = 2_000 + + +class _MaskingGuardrail(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="masking") + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return GenericGuardrailAPIInputs(texts=[text.replace("secret", "[MASKED]") for text in inputs.get("texts", [])]) + + +@pytest.mark.benchmark +def test_extraction_request_guardrail_over_many_field_instructions() -> None: + data: Final = { + "model": "invoice-reader", + "fields": { + f"field_{index}": {"type": "text", "instructions": f"secret {index}"} for index in range(MANY_FIELDS) + }, + } + + result: Final = asyncio.run(ExtractionHandler().process_input_messages(data, _MaskingGuardrail())) + + assert result["fields"] == { + f"field_{index}": {"type": "text", "instructions": f"[MASKED] {index}"} for index in range(MANY_FIELDS) + } + + +@pytest.mark.benchmark +def test_extraction_response_guardrail_over_many_text_answers() -> None: + response: Final = ExtractionResponse.model_validate( + { + "model": "invoice-reader", + "answers": {f"field_{index}": {"type": "text", "value": f"secret {index}"} for index in range(MANY_FIELDS)}, + "truncated": False, + "usage": {"input_tokens": 52, "output_tokens": 0}, + } + ) + + result: Final = asyncio.run(ExtractionHandler().process_output_response(response, _MaskingGuardrail())) + + assert isinstance(result, ExtractionResponse) + assert {name: answer.value for name, answer in result.answers.items()} == { + f"field_{index}": f"[MASKED] {index}" for index in range(MANY_FIELDS) + } diff --git a/tests/unit/extraction/__init__.py b/tests/unit/extraction/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/extraction/test_main.py b/tests/unit/extraction/test_main.py new file mode 100644 index 00000000000..584ed9cbb91 --- /dev/null +++ b/tests/unit/extraction/test_main.py @@ -0,0 +1,422 @@ +from __future__ import annotations + +import asyncio +import json +import re +from collections.abc import Mapping +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import respx + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.types.extraction import ExtractionResponse + +_CLAIRIC_URL: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clairic" +_STATE: Final = "Invoice #4471 from Acme Tools, dated 2026-03-03. 12 hammers. Total $174. Payment due in 30 days." +_FIELDS: Final[Mapping[str, Mapping[str, object]]] = MappingProxyType( + { + "vendor": {"type": "text", "instructions": "Who sent the invoice?"}, + "invoice_date": {"type": "date"}, + "total": {"type": "number", "unit": "USD"}, + "items": {"type": "list", "items": "text"}, + "days_to_pay": {"type": "estimate", "min": 0, "max": 90, "provider_field": "kept"}, + } +) +_INPUT_TOKENS: Final = 416 +_RESULT: Final[Mapping[str, object]] = { + "model": "clairic", + "answers": { + "vendor": { + "type": "text", + "p_null": 0.0177, + "span": [19, 29], + "guess": "Acme Tools", + "confidence": 0.9998, + "value": "Acme Tools", + }, + "invoice_date": {"type": "date", "p_null": 6.4e-07, "confidence": 0.99, "value": "2026-03-03"}, + "total": {"type": "number", "p_null": 4.2e-07, "guess": 174, "confidence": 0.99, "value": 174}, + "items": {"type": "list", "value": ["12", "hammers"], "spans": [[51, 53], [54, 61]], "confidence": 0.07}, + "days_to_pay": { + "type": "estimate", + "value": 31, + "mean": 30.76, + "median": 30, + "p10": 29, + "p90": 34, + "p_below": 1.2e-06, + "p_above": 1.2e-06, + }, + }, + "truncated": False, + "usage": { + "state_tokens": 52, + "image_tokens": 0, + "schema_tokens": 312, + "input_tokens": _INPUT_TOKENS, + "output_tokens": 0, + "passes": 2, + }, +} +_ENVELOPE: Final[Mapping[str, object]] = {"result": _RESULT, "success": True, "errors": [], "messages": []} +_PNG_DATA_URL: Final = "data:image/png;base64,AA==" + + +@pytest.fixture(autouse=True) +def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.fixture +def cloudflare_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + monkeypatch.setenv("CLOUDFLARE_API_KEY", "cloudflare-key") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + + +def _assert_invoice_answers(response: ExtractionResponse) -> None: + assert response.answers["vendor"].value == "Acme Tools" + assert response.answers["vendor"].span == (19, 29) + assert response.answers["items"].spans == ((51, 53), (54, 61)) + assert response.answers["days_to_pay"].median == 30 + assert response.usage.input_tokens == _INPUT_TOKENS + assert response.usage.schema_tokens == 312 + assert response.truncated is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("cloudflare/clairic", "cloudflare/@cf/cloudflare/clairic")) +@pytest.mark.usefixtures("cloudflare_env") +async def test_clairic_forwards_the_caller_body_and_unwraps_the_envelope( + respx_mock: respx.MockRouter, model: str +) -> None: + route: Final = respx_mock.post(_CLAIRIC_URL).respond(json=_ENVELOPE) + + response: Final = await litellm.aextraction( + model=model, + state=_STATE, + fields=_FIELDS, + images=[_PNG_DATA_URL, {"content_type": "image/jpeg", "base64": "AA=="}], + reference_date="2026-03-04", + options={"include_quantiles": True}, + ) + + assert route.call_count == 1 + request: Final = route.calls[0].request + assert request.headers["authorization"] == "Bearer cloudflare-key" + assert json.loads(request.content) == { + "model": "clairic", + "state": _STATE, + "fields": _FIELDS, + "images": [_PNG_DATA_URL, {"content_type": "image/jpeg", "base64": "AA=="}], + "reference_date": "2026-03-04", + "options": {"include_quantiles": True}, + } + _assert_invoice_answers(response) + assert response.model == model + assert response.hidden_params["model"] == "cloudflare/@cf/cloudflare/clairic" + assert response.model_dump(mode="json", exclude={"model"}) == { + key: value for key, value in _RESULT.items() if key != "model" + } + + +@pytest.mark.usefixtures("cloudflare_env") +def test_sync_extraction_reads_an_unwrapped_result(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_CLAIRIC_URL).respond(json=_RESULT) + + response: Final = litellm.extraction(model="cloudflare/clairic", state=_STATE, fields=_FIELDS) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == {"model": "clairic", "state": _STATE, "fields": _FIELDS} + _assert_invoice_answers(response) + + +def _clairic_transport() -> httpx.MockTransport: + def respond(request: httpx.Request) -> httpx.Response: + assert str(request.url) == _CLAIRIC_URL + return httpx.Response(200, json=_ENVELOPE) + + return httpx.MockTransport(respond) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_env") +async def test_async_extraction_sends_through_the_callers_client(respx_mock: respx.MockRouter) -> None: + response: Final = await litellm.aextraction( + model="cloudflare/clairic", + state=_STATE, + fields=_FIELDS, + client=AsyncHTTPHandler(transport=_clairic_transport()), + ) + + assert len(respx_mock.calls) == 0 + _assert_invoice_answers(response) + + +@pytest.mark.usefixtures("cloudflare_env") +def test_sync_extraction_sends_through_the_callers_client(respx_mock: respx.MockRouter) -> None: + response: Final = litellm.extraction( + model="cloudflare/clairic", + state=_STATE, + fields=_FIELDS, + client=HTTPHandler(client=httpx.Client(transport=_clairic_transport())), + ) + + assert len(respx_mock.calls) == 0 + _assert_invoice_answers(response) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_env") +async def test_a_cloudflare_bad_input_is_a_bad_request_with_the_cloudflare_message( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_CLAIRIC_URL).respond( + status_code=400, + json={ + "errors": [{"message": "AiError: Bad input: fields.total.decimals must be 0 to 6", "code": 5006}], + "success": False, + "result": {}, + "messages": [], + }, + ) + + with pytest.raises(litellm.BadRequestError, match=re.escape("fields.total.decimals must be 0 to 6")): + await litellm.aextraction( + model="cloudflare/clairic", state=_STATE, fields={"total": {"type": "number", "decimals": 9}} + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("fields", "images"), + ( + ({}, None), + ({"total": {"type": "money"}}, None), + ({"bad name": {"type": "text"}}, None), + ({"total": {"type": "number"}}, ["https://example.com/invoice.png"]), + ({"total": {"type": "number"}}, [_PNG_DATA_URL] * 5), + ), + ids=("no_fields", "unknown_type", "invalid_field_name", "remote_image", "too_many_images"), +) +@pytest.mark.usefixtures("cloudflare_env") +async def test_an_invalid_request_is_refused_before_http( + respx_mock: respx.MockRouter, fields: Mapping[str, object], images: list[str] | None +) -> None: + with pytest.raises(litellm.BadRequestError, match="Invalid Extraction request"): + await litellm.aextraction(model="cloudflare/clairic", state=_STATE, fields=fields, images=images) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("unset", "message"), + ( + ("CLOUDFLARE_ACCOUNT_ID", "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID"), + ("CLOUDFLARE_API_KEY", "Missing API key for Extraction provider 'cloudflare'"), + ), +) +@pytest.mark.usefixtures("cloudflare_env") +async def test_missing_cloudflare_credentials_are_refused_before_http( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, unset: str, message: str +) -> None: + monkeypatch.delenv(unset) + + with pytest.raises(litellm.BadRequestError, match=message): + await litellm.aextraction(model="cloudflare/clairic", state=_STATE, fields=_FIELDS) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_a_provider_without_extraction_is_refused(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match=r"Unknown Extraction provider 'openai'\. Supported.*cloudflare"): + await litellm.aextraction(model="openai/gpt-5.5", state=_STATE, fields=_FIELDS, api_key="k") + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("model", ("cloudflare/clairic", "cloudflare/@cf/cloudflare/clairic")) +@pytest.mark.usefixtures("cloudflare_env") +async def test_clairic_cost_bills_input_tokens_at_the_cost_map_price(respx_mock: respx.MockRouter, model: str) -> None: + respx_mock.post(_CLAIRIC_URL).respond(json=_ENVELOPE) + + response: Final = await litellm.aextraction(model=model, state=_STATE, fields=_FIELDS) + + expected_cost: Final = _INPUT_TOKENS * float(litellm.model_cost[model]["input_cost_per_token"]) + assert expected_cost > 0 + assert litellm.completion_cost(completion_response=response) == pytest.approx(expected_cost) + assert response.hidden_params["response_cost"] == pytest.approx(expected_cost) + + +@pytest.mark.asyncio +async def test_router_extraction_sends_the_deployment_model_and_credentials(respx_mock: respx.MockRouter) -> None: + router: Final = litellm.Router( + model_list=[ + { + "model_name": "invoice-reader", + "litellm_params": { + "model": "cloudflare/clairic", + "api_key": "deployment-key", + "api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/v1", + "additional_drop_params": ["temperature"], + }, + } + ] + ) + route: Final = respx_mock.post(_CLAIRIC_URL).respond(json=_ENVELOPE) + + response: Final = await router.aextraction(model="invoice-reader", state=_STATE, fields=_FIELDS) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == "Bearer deployment-key" + assert json.loads(route.calls[0].request.content) == {"model": "clairic", "state": _STATE, "fields": _FIELDS} + assert response.model == "cloudflare/clairic" + _assert_invoice_answers(response) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_env") +async def test_litellm_params_stay_local(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_CLAIRIC_URL).respond(json=_ENVELOPE) + + await litellm.aextraction( + model="cloudflare/clairic", + state=_STATE, + fields=_FIELDS, + metadata={"team": "billing"}, + num_retries=0, + stream=False, + additional_drop_params=["language"], + ) + + assert json.loads(route.calls[0].request.content) == {"model": "clairic", "state": _STATE, "fields": _FIELDS} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("name", ("language", "priority", "session_id", "mock_delay", "extra_body")) +@pytest.mark.usefixtures("cloudflare_env") +async def test_a_key_clairic_does_not_take_is_refused_before_http_without_drop_params( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, name: str +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape(f"does not support parameters: ['{name}']")): + await litellm.aextraction(model="cloudflare/clairic", state=_STATE, fields=_FIELDS, **{name: "x"}) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_env") +async def test_additional_drop_params_drops_only_the_keys_it_names( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + route: Final = respx_mock.post(_CLAIRIC_URL).respond(json=_ENVELOPE) + + await litellm.aextraction( + model="cloudflare/clairic", + state=_STATE, + fields=_FIELDS, + temperature=0.2, + language="en", + additional_drop_params=["temperature", "language"], + ) + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['seed']")): + await litellm.aextraction( + model="cloudflare/clairic", state=_STATE, fields=_FIELDS, seed=7, additional_drop_params=["temperature"] + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == {"model": "clairic", "state": _STATE, "fields": _FIELDS} + + +class _RecordingLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.standard_logging_object: Mapping[str, object] | None = None + self.stream: object = None + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self.stream = kwargs.get("stream") + standard_logging_object: Final = kwargs.get("standard_logging_object") + if isinstance(standard_logging_object, dict): + self.standard_logging_object = standard_logging_object + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_env") +async def test_stream_is_refused_before_http_without_drop_params(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['stream']")): + await litellm.aextraction(model="cloudflare/clairic", state=_STATE, fields=_FIELDS, stream=True) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("name", "value"), + (("temperature", 0.2), ("user", "end-user-1"), ("seed", 7), ("response_format", {"type": "json_object"})), +) +@pytest.mark.usefixtures("cloudflare_env") +async def test_openai_generation_params_are_refused_before_http_without_drop_params( + respx_mock: respx.MockRouter, name: str, value: object +) -> None: + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape(f"does not support parameters: ['{name}']")): + await litellm.aextraction(model="cloudflare/clairic", state=_STATE, fields=_FIELDS, **{name: value}) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_env") +async def test_openai_generation_params_are_dropped_with_drop_params(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(_CLAIRIC_URL).respond(json=_ENVELOPE) + + await litellm.aextraction( + model="cloudflare/clairic", + state=_STATE, + fields=_FIELDS, + temperature=0.2, + user="end-user-1", + seed=7, + drop_params=True, + ) + + assert json.loads(route.calls[0].request.content) == {"model": "clairic", "state": _STATE, "fields": _FIELDS} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("cloudflare_env") +async def test_a_dropped_stream_is_sent_and_logged_as_a_billed_non_stream_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + route: Final = respx_mock.post(_CLAIRIC_URL).respond(json=_ENVELOPE) + recording_logger: Final = _RecordingLogger() + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + response: Final = await litellm.aextraction( + model="cloudflare/clairic", state=_STATE, fields=_FIELDS, stream=True, drop_params=True + ) + await asyncio.sleep(0) + GLOBAL_LOGGING_WORKER.start() + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10.0) + + assert "stream" not in json.loads(route.calls[0].request.content) + _assert_invoice_answers(response) + logged: Final = recording_logger.standard_logging_object + assert logged is not None + assert recording_logger.stream is False + assert response.hidden_params["response_cost"] > 0 + assert logged["response_cost"] == pytest.approx(response.hidden_params["response_cost"]) diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index c23749f7a20..b584a72855a 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -195,9 +195,7 @@ def test_update_model_params_with_health_check_tracking_information(): }, } - result = HealthCheckHelpers.update_model_params_with_health_check_tracking_information( - initial_model_params - ) + result = HealthCheckHelpers.update_model_params_with_health_check_tracking_information(initial_model_params) # Verify that litellm_metadata was added assert "litellm_metadata" in result @@ -753,9 +751,7 @@ async def test_ahealth_check_keeps_the_noul_probe_for_other_decisions_providers( assert "error" not in result, result assert upstream.called sent: Final = json.loads(upstream.calls[0].request.content) - assert sent["questions"] == { - "reachable": {"type": "noul", "instructions": "Is the service reachable?"} - } + assert sent["questions"] == {"reachable": {"type": "noul", "instructions": "Is the service reachable?"}} @pytest.mark.parametrize( @@ -916,6 +912,7 @@ async def test_ahealth_check_without_mode_reports_the_real_failure( assert "Missing `mode`" not in result["error"] assert "raw_request_typed_dict" in result + def test_update_litellm_params_for_health_check(): """ Test if update_litellm_params_for_health_check correctly: @@ -1010,12 +1007,10 @@ def test_update_litellm_params_for_health_check(): "model": f"bedrock/{prefix}anthropic.claude-3-haiku-20240307-v1:0", "api_key": "fake_key", } - updated_params = update_litellm_params_for_health_check( - model_info, litellm_params + updated_params = update_litellm_params_for_health_check(model_info, litellm_params) + assert updated_params["model"] == f"{prefix}anthropic.claude-3-haiku-20240307-v1:0", ( + f"Failed to preserve CRIS prefix: {prefix}" ) - assert ( - updated_params["model"] == f"{prefix}anthropic.claude-3-haiku-20240307-v1:0" - ), f"Failed to preserve CRIS prefix: {prefix}" litellm_params = { "model": "bedrock/us-east-2/us.anthropic.claude-3-haiku-20240307-v1:0", @@ -1036,30 +1031,21 @@ def test_update_litellm_params_for_health_check(): "api_key": "fake_key", } updated_params = update_litellm_params_for_health_check(model_info, litellm_params) - assert ( - updated_params["model"] - == "llama/arn:aws:bedrock:us-east-1:123:imported-model/abc" - ) + assert updated_params["model"] == "llama/arn:aws:bedrock:us-east-1:123:imported-model/abc" litellm_params = { "model": "bedrock/deepseek_r1/arn:aws:bedrock:us-west-2:456:imported-model/xyz", "api_key": "fake_key", } updated_params = update_litellm_params_for_health_check(model_info, litellm_params) - assert ( - updated_params["model"] - == "deepseek_r1/arn:aws:bedrock:us-west-2:456:imported-model/xyz" - ) + assert updated_params["model"] == "deepseek_r1/arn:aws:bedrock:us-west-2:456:imported-model/xyz" litellm_params = { "model": "bedrock/converse/us.anthropic.claude-haiku-4-5-20251001-v1:0", "api_key": "fake_key", } updated_params = update_litellm_params_for_health_check(model_info, litellm_params) - assert ( - updated_params["model"] - == "converse/us.anthropic.claude-haiku-4-5-20251001-v1:0" - ) + assert updated_params["model"] == "converse/us.anthropic.claude-haiku-4-5-20251001-v1:0" litellm_params = { "model": "bedrock/invoke/us-west-2/anthropic.claude-instant-v1", @@ -1073,29 +1059,22 @@ def test_update_litellm_params_for_health_check(): "api_key": "fake_key", } updated_params = update_litellm_params_for_health_check(model_info, litellm_params) - assert ( - updated_params["model"] - == "arn:aws:bedrock:eu-central-1:000:application-inference-profile/abc" - ) + assert updated_params["model"] == "arn:aws:bedrock:eu-central-1:000:application-inference-profile/abc" litellm_params = { "model": "bedrock/us-west-2/llama/arn:aws:bedrock:us-east-1:123:imported-model/abc", "api_key": "fake_key", } updated_params = update_litellm_params_for_health_check(model_info, litellm_params) - assert ( - updated_params["model"] - == "llama/arn:aws:bedrock:us-east-1:123:imported-model/abc" - ) + assert updated_params["model"] == "llama/arn:aws:bedrock:us-east-1:123:imported-model/abc" litellm_params = { "model": "bedrock/converse/us-west-2/eu.anthropic.claude-3-sonnet-20240229-v1:0", "api_key": "fake_key", } updated_params = update_litellm_params_for_health_check(model_info, litellm_params) - assert ( - updated_params["model"] == "converse/eu.anthropic.claude-3-sonnet-20240229-v1:0" - ) + assert updated_params["model"] == "converse/eu.anthropic.claude-3-sonnet-20240229-v1:0" + @pytest.mark.asyncio async def test_perform_health_check_filters_by_model_id(): @@ -1142,6 +1121,7 @@ async def test_perform_health_check_filters_by_model_id(): assert len(healthy_endpoints) == 1 assert healthy_endpoints[0]["api_key"] == "fake-key-2" + @pytest.mark.asyncio async def test_perform_health_check_skip_disabled_background_models(): from litellm.proxy.health_check import perform_health_check @@ -1180,6 +1160,7 @@ async def test_perform_health_check_skip_disabled_background_models(): assert len(captured[0]) == 1 assert captured[0][0]["model_name"] == "a" + @pytest.mark.asyncio async def test_perform_health_check_with_health_check_model(): """ @@ -1206,9 +1187,7 @@ async def test_perform_health_check_with_health_check_model(): return {"status": "healthy"} with patch("litellm.ahealth_check", side_effect=mock_health_check): - healthy_endpoints, unhealthy_endpoints, _ = await _perform_health_check( - model_list - ) + healthy_endpoints, unhealthy_endpoints, _ = await _perform_health_check(model_list) print("health check calls: ", health_check_calls) assert health_check_calls[0] == "openai/gpt-5-mini" @@ -1217,6 +1196,7 @@ async def test_perform_health_check_with_health_check_model(): assert len(healthy_endpoints) == 1 assert len(unhealthy_endpoints) == 0 + @pytest.mark.asyncio async def test_image_generation_health_check_prompt(monkeypatch): """Health checks should respect default and environment-configured prompts.""" @@ -1267,9 +1247,7 @@ async def test_image_generation_health_check_prompt(monkeypatch): health_check_calls = await run_health_check(reloaded_health_check) assert len(health_check_calls) == 1 - assert ( - health_check_calls[0]["prompt"] == reloaded_constants.DEFAULT_HEALTH_CHECK_PROMPT - ) + assert health_check_calls[0]["prompt"] == reloaded_constants.DEFAULT_HEALTH_CHECK_PROMPT # Environment override should change the prompt without code changes override_prompt = "environment override prompt" @@ -1293,9 +1271,7 @@ async def test_health_check_with_custom_llm_provider( "object": "chat.completion", "created": 0, "model": "deepseek-r1-distill-qwen-1.5B-q4", - "choices": [ - {"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"} - ], + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, } ) @@ -1503,3 +1479,37 @@ async def test_audio_transcription_health_check_returns_transcribed_text( assert "error" not in response assert upstream.called assert b'name="file"' in upstream.calls.last.request.content + + +@pytest.mark.parametrize("model", ("cloudflare/clairic", "cloudflare/@cf/cloudflare/clairic")) +async def test_ahealth_check_probes_extraction_models_through_the_extraction_api( + model: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct-1") + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post( + "https://api.cloudflare.com/client/v4/accounts/acct-1/ai/run/@cf/cloudflare/clairic" + ).respond( + json={ + "result": { + "model": "clairic", + "answers": {"reachable": {"type": "text", "value": "health"}}, + "truncated": False, + "usage": {"input_tokens": 20, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + } + ) + + result: Final = await ahealth_check({"model": model, "api_key": "sk-test"}, mode=None) + + assert "error" not in result, result + sent: Final = json.loads(upstream.calls[0].request.content) + assert sent["state"] == "health check" + assert sent["fields"]["reachable"]["type"] == "text" diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index 41ad27d6f40..b3bd1b8743a 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -477,6 +477,7 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client(): "asearch", "adecisions", "asystemone", + "aextraction", "avector_store_create", "avector_store_search", "acreate_skill", diff --git a/tests/unit/llms/cloudflare/test_cloudflare_extraction_guardrail_handler.py b/tests/unit/llms/cloudflare/test_cloudflare_extraction_guardrail_handler.py new file mode 100644 index 00000000000..1db105f4495 --- /dev/null +++ b/tests/unit/llms/cloudflare/test_cloudflare_extraction_guardrail_handler.py @@ -0,0 +1,335 @@ +from typing import Final, Literal + +import pytest + +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms import get_guardrail_translation_mapping +from litellm.llms.cloudflare.extraction.guardrail_translation.handler import ExtractionHandler +from litellm.types.extraction import ExtractionResponse +from litellm.types.utils import CallTypes, GenericGuardrailAPIInputs + + +class _TaggingGuardrail(CustomGuardrail): + def __init__(self) -> None: + super().__init__(guardrail_name="tagging") + self.seen: tuple[str, tuple[str, ...]] | None = None + self.seen_images: tuple[str, ...] = () + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + texts: Final = tuple(inputs.get("texts", [])) + self.seen = (input_type, texts) + self.seen_images = tuple(inputs.get("images", [])) + return GenericGuardrailAPIInputs(texts=[f"{text} [GUARDRAILED]" for text in texts]) + + +class _MaskingGuardrail(CustomGuardrail): + def __init__(self, secret: str, mask: str) -> None: + super().__init__(guardrail_name="masking") + self.secret = secret + self.mask = mask + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return GenericGuardrailAPIInputs( + texts=[text.replace(self.secret, self.mask) for text in inputs.get("texts", [])] + ) + + +@pytest.mark.parametrize("call_type", (CallTypes.extraction, CallTypes.aextraction)) +def test_extraction_call_types_resolve_to_the_extraction_handler(call_type: CallTypes) -> None: + assert get_guardrail_translation_mapping(call_type) is ExtractionHandler + + +@pytest.mark.asyncio +async def test_request_state_and_field_instructions_are_guardrailed_on_a_copy() -> None: + guardrail: Final = _TaggingGuardrail() + data: Final = { + "model": "invoice-reader", + "state": "Invoice from Acme Tools, contact jane@acme.example", + "fields": { + "vendor": {"type": "text", "instructions": "Who sent the invoice?"}, + "total": {"type": "number", "unit": "USD"}, + }, + "images": ["data:image/png;base64,AA=="], + } + + result: Final = await ExtractionHandler().process_input_messages(data, guardrail) + + assert guardrail.seen == ( + "request", + ("Invoice from Acme Tools, contact jane@acme.example", "Who sent the invoice?", "USD"), + ) + assert {key: value for key, value in result.items() if key != "metadata"} == { + **{key: value for key, value in data.items() if key != "metadata"}, + "state": "Invoice from Acme Tools, contact jane@acme.example [GUARDRAILED]", + "fields": { + "vendor": {"type": "text", "instructions": "Who sent the invoice? [GUARDRAILED]"}, + "total": {"type": "number", "unit": "USD [GUARDRAILED]"}, + }, + } + assert data["state"] == "Invoice from Acme Tools, contact jane@acme.example" + + +@pytest.mark.asyncio +async def test_a_request_with_no_text_or_images_skips_the_guardrail() -> None: + guardrail: Final = _TaggingGuardrail() + data: Final = {"model": "invoice-reader", "fields": {"total": {"type": "number"}}} + + assert await ExtractionHandler().process_input_messages(data, guardrail) is data + assert guardrail.seen is None + + +@pytest.mark.asyncio +async def test_request_images_reach_the_guardrail_as_data_urls() -> None: + guardrail: Final = _TaggingGuardrail() + data: Final = { + "model": "invoice-reader", + "fields": {"total": {"type": "number"}}, + "images": [{"content_type": "image/png", "base64": "iVBORw0KGgo="}, "data:image/jpeg;base64,/9j/"], + } + + result: Final = await ExtractionHandler().process_input_messages(data, guardrail) + + assert guardrail.seen == ("request", ()) + assert guardrail.seen_images == ("data:image/png;base64,iVBORw0KGgo=", "data:image/jpeg;base64,/9j/") + assert result["images"] == data["images"] + + +@pytest.mark.asyncio +async def test_response_answers_and_state_text_are_guardrailed_on_a_copy_without_stale_spans() -> None: + guardrail: Final = _TaggingGuardrail() + response: Final = ExtractionResponse.model_validate( + { + "model": "invoice-reader", + "answers": { + "vendor": {"type": "text", "value": "Acme Tools", "guess": "Acme Tools", "span": [19, 29]}, + "items": {"type": "list", "value": ["12", "hammers"], "spans": [[51, 53], [54, 61]]}, + "total": {"type": "number", "value": 174, "guess": 174}, + "contact": {"type": "text", "value": None, "p_null": 0.98}, + }, + "truncated": False, + "state_text": "Invoice from Acme Tools", + "usage": {"input_tokens": 52, "output_tokens": 0}, + } + ) + response.set_hidden_params({"response_cost": 0.0000052}) + + result: Final = await ExtractionHandler().process_output_response(response, guardrail) + + assert guardrail.seen == ( + "response", + ("Acme Tools", "12", "hammers", "174", "Invoice from Acme Tools"), + ) + assert isinstance(result, ExtractionResponse) + assert result.model_dump(mode="json") == { + **response.model_dump(mode="json"), + "answers": { + "vendor": {"type": "text", "value": "Acme Tools [GUARDRAILED]", "guess": "Acme Tools [GUARDRAILED]"}, + "items": {"type": "list", "value": ["12 [GUARDRAILED]", "hammers [GUARDRAILED]"]}, + "total": {"type": "number", "value": None}, + "contact": {"type": "text", "value": None, "p_null": 0.98}, + }, + "state_text": "Invoice from Acme Tools [GUARDRAILED]", + } + assert response.answers["vendor"].value == "Acme Tools" + assert result.hidden_params["response_cost"] == 0.0000052 + + +@pytest.mark.asyncio +async def test_spans_into_the_callers_own_state_survive_a_response_guardrail() -> None: + response: Final = ExtractionResponse.model_validate( + { + "model": "invoice-reader", + "answers": {"vendor": {"type": "text", "value": "Acme Tools", "span": [19, 29]}}, + "truncated": False, + "usage": {"input_tokens": 52, "output_tokens": 0}, + } + ) + + result: Final = await ExtractionHandler().process_output_response(response, _TaggingGuardrail()) + + assert isinstance(result, ExtractionResponse) + assert result.answers["vendor"].model_dump(mode="json") == { + "type": "text", + "value": "Acme Tools [GUARDRAILED]", + "span": [19, 29], + } + + +@pytest.mark.asyncio +async def test_a_response_guardrail_change_to_a_typed_answer_nulls_it_instead_of_breaking_its_type() -> None: + guardrail: Final = _MaskingGuardrail("2026-03-03", "[DATE]") + request_data: Final = { + "fields": { + "invoice_date": {"type": "date"}, + "due_dates": {"type": "list", "items": "date"}, + "notes": {"type": "list", "items": "text"}, + } + } + response: Final = ExtractionResponse.model_validate( + { + "model": "invoice-reader", + "answers": { + "invoice_date": {"type": "date", "value": "2026-03-03", "guess": "2026-03-03", "span": [30, 40]}, + "due_dates": {"type": "list", "value": ["2026-03-03", "2026-04-02"], "spans": [[30, 40], [50, 60]]}, + "notes": {"type": "list", "value": ["paid 2026-03-03"], "spans": [[70, 85]]}, + "received": {"type": "date", "value": "2026-02-27", "span": [90, 100]}, + }, + "truncated": False, + "usage": {"input_tokens": 52, "output_tokens": 0}, + } + ) + + result: Final = await ExtractionHandler().process_output_response(response, guardrail, request_data=request_data) + + assert isinstance(result, ExtractionResponse) + assert {name: answer.model_dump(mode="json") for name, answer in result.answers.items()} == { + "invoice_date": {"type": "date", "value": None}, + "due_dates": {"type": "list", "value": None}, + "notes": {"type": "list", "value": ["paid [DATE]"], "spans": [[70, 85]]}, + "received": {"type": "date", "value": "2026-02-27", "span": [90, 100]}, + } + + +@pytest.mark.asyncio +async def test_a_response_guardrail_that_masks_a_number_nulls_every_answer_carrying_it() -> None: + request_data: Final = { + "fields": { + "phone_text": {"type": "text"}, + "phone_number": {"type": "number"}, + "phone_list": {"type": "list", "items": "number"}, + "phone_mean": {"type": "estimate"}, + "phone_median": {"type": "estimate"}, + "phone_p10": {"type": "estimate"}, + "phone_p90": {"type": "estimate"}, + "phone_quantile": {"type": "estimate"}, + "phone_forecast": {"type": "forecast"}, + "total": {"type": "number"}, + } + } + response: Final = ExtractionResponse.model_validate( + { + "model": "invoice-reader", + "answers": { + "phone_text": {"type": "text", "value": "4155550123", "span": [10, 20]}, + "phone_number": {"type": "number", "value": 4155550123, "guess": 4155550123, "span": [10, 20]}, + "phone_list": {"type": "list", "value": [4155550123], "spans": [[10, 20]]}, + "phone_mean": { + "type": "estimate", + "value": None, + "mean": 4155550123.4, + "p10": 4155550000, + "p_null": 0.6, + }, + "phone_median": { + "type": "estimate", + "value": None, + "median": 4155550123, + "p90": 4155550200, + "p_null": 0.6, + }, + "phone_p10": {"type": "estimate", "value": None, "p10": 4155550123, "p_null": 0.6}, + "phone_p90": {"type": "estimate", "value": None, "p90": 4155550123.2, "p_null": 0.6}, + "phone_quantile": { + "type": "estimate", + "value": None, + "quantiles": [4155550000, 4155550123, 4155550200], + "p_null": 0.6, + }, + "phone_forecast": { + "type": "forecast", + "value": None, + "steps": [{"value": 12, "p10": 10}, {"value": 13, "p90": 4155550123}], + "p_null": 0.6, + }, + "total": {"type": "number", "value": 174, "guess": 174}, + }, + "truncated": False, + "usage": {"input_tokens": 52, "output_tokens": 0}, + } + ) + + result: Final = await ExtractionHandler().process_output_response( + response, _MaskingGuardrail("4155550123", "[PHONE]"), request_data=request_data + ) + + assert isinstance(result, ExtractionResponse) + assert {name: answer.model_dump(mode="json") for name, answer in result.answers.items()} == { + "phone_text": {"type": "text", "value": "[PHONE]", "span": [10, 20]}, + "phone_number": {"type": "number", "value": None}, + "phone_list": {"type": "list", "value": None}, + "phone_mean": {"type": "estimate", "value": None, "p_null": 0.6}, + "phone_median": {"type": "estimate", "value": None, "p_null": 0.6}, + "phone_p10": {"type": "estimate", "value": None, "p_null": 0.6}, + "phone_p90": {"type": "estimate", "value": None, "p_null": 0.6}, + "phone_quantile": {"type": "estimate", "value": None, "p_null": 0.6}, + "phone_forecast": {"type": "forecast", "value": None, "p_null": 0.6}, + "total": {"type": "number", "value": 174, "guess": 174}, + } + + +@pytest.mark.asyncio +async def test_a_response_guardrail_does_not_scan_the_fraction_digits_of_an_estimate() -> None: + days_late: Final = { + "type": "estimate", + "value": None, + "mean": 3.4525830151341586, + "median": 2.4525830151341586, + "p_null": 0.2, + } + response: Final = ExtractionResponse.model_validate( + { + "model": "invoice-reader", + "answers": {"days_late": days_late}, + "truncated": False, + "usage": {"input_tokens": 52, "output_tokens": 0}, + } + ) + + result: Final = await ExtractionHandler().process_output_response( + response, + _MaskingGuardrail("4525830151341586", "[CARD]"), + request_data={"fields": {"days_late": {"type": "estimate"}}}, + ) + + assert isinstance(result, ExtractionResponse) + assert result.answers["days_late"].model_dump(mode="json") == days_late + + +@pytest.mark.asyncio +async def test_a_response_guardrail_sees_the_decimal_text_of_a_fractional_number_answer() -> None: + response: Final = ExtractionResponse.model_validate( + { + "model": "invoice-reader", + "answers": { + "unit_price": {"type": "number", "value": 14.5, "guess": 14.5, "span": [30, 36]}, + "prices": {"type": "list", "value": [14.5, 2.25], "spans": [[30, 36], [40, 45]]}, + }, + "truncated": False, + "usage": {"input_tokens": 52, "output_tokens": 0}, + } + ) + + result: Final = await ExtractionHandler().process_output_response( + response, + _MaskingGuardrail("14.5", "[PRICE]"), + request_data={"fields": {"unit_price": {"type": "number"}, "prices": {"type": "list", "items": "number"}}}, + ) + + assert isinstance(result, ExtractionResponse) + assert {name: answer.model_dump(mode="json") for name, answer in result.answers.items()} == { + "unit_price": {"type": "number", "value": None}, + "prices": {"type": "list", "value": None}, + } diff --git a/tests/unit/llms/cloudflare/test_cloudflare_extraction_transformation.py b/tests/unit/llms/cloudflare/test_cloudflare_extraction_transformation.py new file mode 100644 index 00000000000..214fec07284 --- /dev/null +++ b/tests/unit/llms/cloudflare/test_cloudflare_extraction_transformation.py @@ -0,0 +1,221 @@ +import json +from typing import Final + +import httpx +import pytest +from pydantic import ValidationError + +import litellm +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.extraction.transformation import BaseExtractionConfig +from litellm.types.extraction import ExtractionRequestBody +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +INVOICE_STATE: Final = "Invoice #4471 from Acme Tools, dated March 3 2026. Total $174.00. Payment due in 30 days." +INVOICE_FIELDS: Final = { + "vendor": {"type": "text", "instructions": "Who sent the invoice"}, + "invoice_date": {"type": "date"}, + "total": {"type": "number", "decimals": 2, "unit": "USD"}, + "items": {"type": "list", "items": "text"}, + "days_to_pay": {"type": "estimate", "min": 0, "max": 90, "step": 1}, +} +CLAIRIC_RESULT: Final = { + "model": "clairic", + "answers": { + "vendor": { + "type": "text", + "p_null": 0.01, + "span": [19, 29], + "guess": "Acme Tools", + "confidence": 0.99, + "value": "Acme Tools", + }, + "total": {"type": "number", "p_null": 0.0, "guess": 174, "confidence": 0.99, "value": 174}, + "items": {"type": "list", "value": ["hammers"], "spans": [[54, 61]], "confidence": 0.4}, + "days_to_pay": { + "type": "estimate", + "value": 31, + "mean": 30.7, + "median": 30, + "p10": 29, + "p90": 34, + "p_below": 0.0, + "p_above": 0.0, + }, + }, + "truncated": False, + "usage": { + "state_tokens": 52, + "image_tokens": 0, + "schema_tokens": 312, + "input_tokens": 416, + "output_tokens": 0, + "passes": 2, + }, +} + + +def _cloudflare_config() -> BaseExtractionConfig: + config: Final = ProviderConfigManager.get_provider_extraction_config("clairic", LlmProviders.CLOUDFLARE) + assert config is not None + return config + + +def _response(payload: object) -> httpx.Response: + return httpx.Response(status_code=200, content=json.dumps(payload).encode()) + + +def test_only_cloudflare_serves_extraction() -> None: + assert ProviderConfigManager.get_provider_extraction_config("gpt-5", LlmProviders.OPENAI) is None + assert isinstance(_cloudflare_config(), litellm.CloudflareExtractionConfig) + + +@pytest.mark.parametrize("model", ("clairic", "@cf/cloudflare/clairic")) +def test_clairic_resolves_to_its_workers_ai_run_url(monkeypatch: pytest.MonkeyPatch, model: str) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + config: Final = _cloudflare_config() + canonical: Final = config.canonical_model(model) + api_base: Final = config.resolve_api_base(None) + + assert canonical == "@cf/cloudflare/clairic" + assert api_base is not None + assert ( + config.get_complete_url(api_base, canonical) + == "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clairic" + ) + + +@pytest.mark.parametrize( + "api_base", + ( + "https://gateway.example/client/v4/accounts/acct/ai/v1", + "https://gateway.example/client/v4/accounts/acct/ai/run/", + "https://gateway.example/client/v4/accounts/acct", + ), +) +def test_custom_api_base_keeps_the_run_path(api_base: str) -> None: + assert ( + _cloudflare_config().get_complete_url(api_base, "@cf/cloudflare/clairic") + == "https://gateway.example/client/v4/accounts/acct/ai/run/@cf/cloudflare/clairic" + ) + + +def test_missing_account_id_leaves_no_default_api_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("CLOUDFLARE_ACCOUNT_ID", raising=False) + monkeypatch.delenv("CLOUDFLARE_API_BASE", raising=False) + + assert _cloudflare_config().resolve_api_base(None) is None + + +def test_request_forwards_the_callers_fields_unchanged() -> None: + request: Final = ExtractionRequestBody.model_validate( + { + "state": INVOICE_STATE, + "fields": INVOICE_FIELDS, + "reference_date": "2026-03-01", + "options": {"return_state_text": True}, + "images": [{"content_type": "image/png", "base64": "iVBORw0KGgo="}], + } + ) + + assert _cloudflare_config().transform_extraction_request("@cf/cloudflare/clairic", request) == { + "model": "clairic", + "state": INVOICE_STATE, + "fields": INVOICE_FIELDS, + "reference_date": "2026-03-01", + "options": {"return_state_text": True}, + "images": [{"content_type": "image/png", "base64": "iVBORw0KGgo="}], + } + + +@pytest.mark.parametrize("wrapped", (True, False)) +def test_response_parses_typed_answers_and_usage(wrapped: bool) -> None: + payload: Final = ( + {"result": CLAIRIC_RESULT, "success": True, "errors": [], "messages": []} if wrapped else CLAIRIC_RESULT + ) + + response: Final = _cloudflare_config().transform_extraction_response( + "@cf/cloudflare/clairic", "cloudflare", _response(payload) + ) + + assert response.model_dump(mode="json") == CLAIRIC_RESULT + assert response.answers["vendor"].span == (19, 29) + assert response.usage.input_tokens == 416 + + +def test_unexpected_response_raises_a_provider_error() -> None: + with pytest.raises(BaseLLMException) as error: + _cloudflare_config().transform_extraction_response( + "@cf/cloudflare/clairic", "cloudflare", _response({"result": {"answers": {}}}) + ) + + assert error.value.status_code == 500 + + +def test_unknown_field_type_is_rejected() -> None: + with pytest.raises(ValidationError, match=r"fields\.vendor\.type"): + ExtractionRequestBody.model_validate({"state": INVOICE_STATE, "fields": {"vendor": {"type": "color"}}}) + + +def test_a_list_field_without_an_item_type_is_rejected_locally() -> None: + # Cloudflare answered a list field without items with 422 code 3030 on @cf/cloudflare/clairic, probed 2026-10-20 + with pytest.raises(ValidationError, match=r"fields\.line_items\n.*list fields need items"): + ExtractionRequestBody.model_validate({"state": INVOICE_STATE, "fields": {"line_items": {"type": "list"}}}) + + +@pytest.mark.parametrize( + "spec", + ( + {"type": "number", "decimals": "2"}, + {"type": "number", "integer": "yes"}, + {"type": "number", "integer": 1}, + {"type": "estimate", "min": True, "max": 90}, + ), + ids=("string_decimals", "string_integer", "number_integer", "bool_min"), +) +def test_a_field_value_of_the_wrong_json_type_is_rejected_instead_of_coerced(spec: dict[str, object]) -> None: + # Cloudflare answered each of these specs with a 400 or 422 on @cf/cloudflare/clairic, probed 2026-10-20 + with pytest.raises(ValidationError, match=r"fields\.total\."): + ExtractionRequestBody.model_validate({"state": INVOICE_STATE, "fields": {"total": spec}}) + + +def test_numbers_in_a_field_spec_go_to_cloudflare_exactly_as_sent() -> None: + # Cloudflare answered "decimals": 2.0 and "step": 1.0 with a 200 on @cf/cloudflare/clairic, probed 2026-10-20 + fields: Final = { + "total": {"type": "number", "decimals": 2.0, "null_threshold": 1}, + "days_to_pay": {"type": "estimate", "min": 0, "max": 90, "step": 1.0}, + } + request: Final = ExtractionRequestBody.model_validate({"state": INVOICE_STATE, "fields": fields}) + + body: Final = _cloudflare_config().transform_extraction_request("@cf/cloudflare/clairic", request) + + assert json.dumps(body["fields"]) == json.dumps(fields) + + +@pytest.mark.parametrize("name", ("due date", "1st_item", "x" * 80), ids=("space", "digit_first", "80_chars")) +def test_a_field_name_clairic_refuses_is_rejected_locally(name: str) -> None: + # Cloudflare answered each of these names with 422 code 3030 on @cf/cloudflare/clairic, probed 2026-10-19 + with pytest.raises(ValidationError, match=r"fields\."): + ExtractionRequestBody.model_validate({"state": INVOICE_STATE, "fields": {name: {"type": "text"}}}) + + +@pytest.mark.parametrize("name", ("_vendor", "line.items", "days-to-pay", "x" * 64)) +def test_field_names_with_dots_dashes_and_underscores_up_to_64_chars_are_accepted(name: str) -> None: + # Cloudflare answered each of these names with a 200 on @cf/cloudflare/clairic, probed 2026-10-19 + request: Final = ExtractionRequestBody.model_validate({"state": INVOICE_STATE, "fields": {name: {"type": "text"}}}) + + assert tuple(request.fields) == (name,) + + +def test_decisions_config_still_resolves_clef_after_the_shared_helpers_moved(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CLOUDFLARE_ACCOUNT_ID", "acct") + config: Final = litellm.CloudflareDecisionsConfig() + api_base: Final = config.get_default_api_base() + + assert api_base is not None + assert ( + config.get_complete_url(api_base, config.canonical_model("clef")) + == "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clef" + ) + assert config.unwrap_response({"result": {"answers": {}}}) == {"answers": {}} diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index fcf56eaea9a..021eb289ad7 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -8,7 +8,7 @@ import sys from collections.abc import AsyncGenerator, Callable, Iterator, Mapping from contextlib import asynccontextmanager from pathlib import Path -from typing import Final +from typing import Final, Literal import pytest import respx @@ -19,6 +19,8 @@ from pydantic import TypeAdapter from starlette.routing import Match import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.decisions_pricing import apply_decisions_pricing from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features from litellm.proxy._types import UserAPIKeyAuth @@ -32,6 +34,9 @@ from litellm.proxy.proxy_server import ( initialize, ) from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody +from litellm.types.extraction import ExtractionRequestBody +from litellm.types.guardrails import GuardrailEventHooks +from litellm.types.utils import GenericGuardrailAPIInputs from tests._master_key import MASTER_KEY if sys.version_info >= (3, 14): @@ -1296,6 +1301,328 @@ def test_the_two_decisions_routes_document_different_request_formats(decisions_o assert "input" in properties("/v1/decisions") and "input" not in properties("/v1/systemone") +_CLAIRIC_UPSTREAM: Final = "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clairic" +_CLAIRIC_INPUT_TOKENS: Final = 416 +_CLAIRIC_RESULT: Final[Mapping[str, object]] = { + "model": "clairic", + "answers": { + "vendor": {"type": "text", "p_null": 0.0177, "span": [19, 29], "confidence": 0.9998, "value": "Acme Tools"}, + "days_to_pay": {"type": "estimate", "value": 31, "mean": 30.76, "median": 30, "p10": 29, "p90": 34}, + }, + "truncated": False, + "usage": {"input_tokens": _CLAIRIC_INPUT_TOKENS, "output_tokens": 0, "schema_tokens": 312, "passes": 2}, +} +_EXTRACTION_REQUEST: Final[Mapping[str, object]] = { + "model": "invoice-reader", + "state": "Invoice #4471 from Acme Tools. Payment due in 30 days.", + "fields": { + "vendor": {"type": "text", "instructions": "Who sent the invoice?"}, + "days_to_pay": {"type": "estimate", "min": 0, "max": 90}, + }, + "images": ["data:image/png;base64,AA=="], +} + + +def _route_invoice_reader_to_clairic( + monkeypatch: pytest.MonkeyPatch, drop_params: bool = False, additional_drop_params: tuple[str, ...] = () +) -> None: + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "invoice-reader", + "litellm_params": { + "model": "cloudflare/clairic", + "api_key": "cloudflare-key", + "api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run", + "drop_params": drop_params, + **({"additional_drop_params": list(additional_drop_params)} if additional_drop_params else {}), + }, + } + ] + ), + ) + + +@pytest.mark.parametrize("endpoint", ("/v1/extraction", "/extraction")) +def test_proxy_extraction_forwards_the_body_to_clairic_and_returns_its_answers_and_cost( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, +) -> None: + _route_invoice_reader_to_clairic(monkeypatch) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond( + json={"result": _CLAIRIC_RESULT, "success": True, "errors": [], "messages": []} + ) + + response: Final = client.post(endpoint, json=_EXTRACTION_REQUEST) + + assert response.status_code == 200, response.text + assert response.json() == {**_CLAIRIC_RESULT, "model": "invoice-reader"} + expected_cost: Final = _CLAIRIC_INPUT_TOKENS * float( + litellm.model_cost["cloudflare/clairic"]["input_cost_per_token"] + ) + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + assert json.loads(upstream.calls[0].request.content) == {**_EXTRACTION_REQUEST, "model": "clairic"} + assert upstream.calls[0].request.headers["authorization"] == "Bearer cloudflare-key" + + +@pytest.mark.parametrize( + "request_body", + ( + {"model": "invoice-reader", "state": "Invoice #4471"}, + {"model": "invoice-reader", "state": "Invoice #4471", "fields": {"total": {"type": "money"}}}, + {**_EXTRACTION_REQUEST, "images": ["https://example.com/invoice.png"]}, + ), + ids=("missing_fields", "unknown_field_type", "remote_image"), +) +def test_proxy_extraction_refuses_a_malformed_body_before_calling_clairic( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + request_body: Mapping[str, object], +) -> None: + _route_invoice_reader_to_clairic(monkeypatch) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond(json=_CLAIRIC_RESULT) + + response: Final = client.post("/v1/extraction", json=request_body) + + assert response.status_code == 400, response.text + assert "Invalid Extraction request" in response.text + assert not upstream.called + + +@pytest.mark.parametrize( + ("key", "value"), + ( + ("language", "en"), + ("priority", 1), + ("extra_body", {"language": "en"}), + ("temperature", 0.5), + ("user", "end-user-1"), + ), +) +def test_proxy_extraction_refuses_a_key_clairic_does_not_take_before_calling_clairic( + client: TestClient, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, key: str, value: object +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + _route_invoice_reader_to_clairic(monkeypatch) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond(json={"result": _CLAIRIC_RESULT, "success": True}) + + response: Final = client.post("/v1/extraction", json={**_EXTRACTION_REQUEST, key: value}) + + assert response.status_code == 400, response.text + assert f"does not support parameters: ['{key}']" in response.text + assert not upstream.called + + +@pytest.mark.parametrize( + ("drop_params", "additional_drop_params"), + ((True, ()), (False, ("language", "temperature", "user"))), + ids=("drop_params", "additional_drop_params"), +) +def test_proxy_extraction_sends_only_the_clairic_request_when_the_deployment_drops_other_keys( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + drop_params: bool, + additional_drop_params: tuple[str, ...], +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + _route_invoice_reader_to_clairic(monkeypatch, drop_params, additional_drop_params) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond(json={"result": _CLAIRIC_RESULT, "success": True}) + + response: Final = client.post( + "/v1/extraction", json={**_EXTRACTION_REQUEST, "language": "en", "temperature": 0.2, "user": "end-user-1"} + ) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == {**_EXTRACTION_REQUEST, "model": "clairic"} + + +def test_proxy_extraction_leaves_out_the_cli_generation_defaults( + client: TestClient, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr(litellm.proxy.proxy_server, "user_max_tokens", 256) + monkeypatch.setattr(litellm.proxy.proxy_server, "user_temperature", 0.2) + _route_invoice_reader_to_clairic(monkeypatch) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond(json={"result": _CLAIRIC_RESULT, "success": True}) + + response: Final = client.post("/v1/extraction", json=_EXTRACTION_REQUEST) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == {**_EXTRACTION_REQUEST, "model": "clairic"} + + +class _PreCallRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.calls = 0 + + async def async_pre_call_hook( + self, user_api_key_dict: UserAPIKeyAuth, cache: object, data: dict, call_type: str + ) -> dict: + self.calls += 1 + return data + + +def test_proxy_extraction_refuses_a_remote_image_before_any_pre_call_hook_runs( + client: TestClient, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + recorder: Final = _PreCallRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + _route_invoice_reader_to_clairic(monkeypatch) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond(json={"result": _CLAIRIC_RESULT, "success": True}) + + response: Final = client.post( + "/v1/extraction", json={**_EXTRACTION_REQUEST, "images": ["https://example.com/invoice.png"]} + ) + + assert response.status_code == 400, response.text + assert recorder.calls == 0 + assert not upstream.called + + +def test_proxy_extraction_refuses_stream_before_calling_clairic_unless_the_deployment_drops_params( + client: TestClient, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + _route_invoice_reader_to_clairic(monkeypatch) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond(json={"result": _CLAIRIC_RESULT, "success": True}) + + response: Final = client.post("/v1/extraction", json={**_EXTRACTION_REQUEST, "stream": True}) + + assert response.status_code == 400, response.text + assert "does not support parameters: ['stream']" in response.text + assert not upstream.called + + +def test_proxy_extraction_answers_a_dropped_stream_with_one_json_body( + client: TestClient, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + _route_invoice_reader_to_clairic(monkeypatch, drop_params=True) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond(json={"result": _CLAIRIC_RESULT, "success": True}) + + response: Final = client.post("/v1/extraction", json={**_EXTRACTION_REQUEST, "stream": True}) + + assert response.status_code == 200, response.text + assert response.headers["content-type"].startswith("application/json") + assert response.json() == {**_CLAIRIC_RESULT, "model": "invoice-reader"} + assert json.loads(upstream.calls[0].request.content) == {**_EXTRACTION_REQUEST, "model": "clairic"} + + +class _RequestEmailMasker(CustomGuardrail): + def __init__(self, email: str) -> None: + super().__init__(guardrail_name="mask-email", event_hook=GuardrailEventHooks.pre_call, default_on=True) + self.email = email + + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: object = None, + ) -> GenericGuardrailAPIInputs: + return GenericGuardrailAPIInputs( + texts=[text.replace(self.email, "[EMAIL]") for text in inputs.get("texts", [])] + ) + + +_EMAIL_STATE: Final = "Questions go to billing@acme.example today. The vendor is Acme Tools." +_MASKED_EMAIL_STATE: Final = "Questions go to [EMAIL] today. The vendor is Acme Tools." +_VENDOR_IN_MASKED_STATE: Final = { + "type": "text", + "value": "Acme Tools", + "span": [_MASKED_EMAIL_STATE.index("Acme Tools"), _MASKED_EMAIL_STATE.index("Acme Tools") + len("Acme Tools")], +} + + +def _clairic_vendor_result(state_text: str | None = None) -> Mapping[str, object]: + return { + "model": "clairic", + "answers": {"vendor": _VENDOR_IN_MASKED_STATE}, + "truncated": False, + "usage": {"input_tokens": 52, "output_tokens": 0}, + **({"state_text": state_text} if state_text is not None else {}), + } + + +@pytest.mark.parametrize( + ("guardrail_email", "sent_state", "returned_vendor"), + ( + ("billing@acme.example", _MASKED_EMAIL_STATE, {"type": "text", "value": "Acme Tools"}), + ("nobody@example.com", _EMAIL_STATE, _VENDOR_IN_MASKED_STATE), + ), + ids=("state_masked", "state_untouched"), +) +def test_proxy_extraction_drops_spans_into_a_state_only_its_request_guardrail_saw( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + guardrail_email: str, + sent_state: str, + returned_vendor: Mapping[str, object], +) -> None: + monkeypatch.setattr(litellm, "callbacks", [_RequestEmailMasker(guardrail_email)]) + _route_invoice_reader_to_clairic(monkeypatch) + upstream: Final = respx_mock.post(_CLAIRIC_UPSTREAM).respond( + json={"result": _clairic_vendor_result(), "success": True} + ) + + response: Final = client.post( + "/v1/extraction", + json={"model": "invoice-reader", "state": _EMAIL_STATE, "fields": {"vendor": {"type": "text"}}}, + ) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content)["state"] == sent_state + assert response.json()["answers"] == {"vendor": returned_vendor} + + +def test_proxy_extraction_keeps_spans_into_the_masked_state_it_returns( + client: TestClient, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "callbacks", [_RequestEmailMasker("billing@acme.example")]) + _route_invoice_reader_to_clairic(monkeypatch) + respx_mock.post(_CLAIRIC_UPSTREAM).respond( + json={"result": _clairic_vendor_result(state_text=_MASKED_EMAIL_STATE), "success": True} + ) + + response: Final = client.post( + "/v1/extraction", + json={ + "model": "invoice-reader", + "state": _EMAIL_STATE, + "fields": {"vendor": {"type": "text"}}, + "options": {"return_state_text": True}, + }, + ) + + assert response.status_code == 200, response.text + body: Final = response.json() + assert body["answers"] == {"vendor": _VENDOR_IN_MASKED_STATE} + assert body["state_text"][slice(*body["answers"]["vendor"]["span"])] == "Acme Tools" + + +@pytest.mark.parametrize("path", ("/v1/extraction", "/extraction")) +def test_swagger_documents_the_extraction_request_and_response(decisions_openapi: dict[str, object], path: str) -> None: + operation = decisions_openapi["paths"][path]["post"] + media = operation["requestBody"]["content"]["application/json"] + + assert {"model", "fields"} <= set(media["schema"]["required"]) + assert "$ref" not in json.dumps(media["schema"]) + example = {key: value for key, value in media["example"].items() if key != "model"} + assert len(TypeAdapter(ExtractionRequestBody).validate_python(example).fields) == 5 + response_ref = operation["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + assert response_ref == "#/components/schemas/ExtractionResponse" + assert {"answers", "usage"} <= set(decisions_openapi["components"]["schemas"]["ExtractionResponse"]["properties"]) + + def test_proxy_systemone_reads_a_zstd_compressed_request_body( client: TestClient, respx_mock: respx.MockRouter, diff --git a/tests/unit/proxy/health_endpoints/test_health_endpoints.py b/tests/unit/proxy/health_endpoints/test_health_endpoints.py index d5288a7fc12..950d1742d49 100644 --- a/tests/unit/proxy/health_endpoints/test_health_endpoints.py +++ b/tests/unit/proxy/health_endpoints/test_health_endpoints.py @@ -5297,9 +5297,49 @@ def test_test_model_connection_accepts_image_edit_mode(monkeypatch): assert response.json()["status"] == "success" -@pytest.mark.parametrize("mode", ("decisions", "evaluation"), ids=("decisions", "legacy-evaluation")) -def test_test_model_connection_accepts_decision_modes( - monkeypatch, mode: Literal["evaluation", "decisions"] +_TYPESAFE_TEST_CONNECTION: Final = ( + {"model": "typesafe/jev-latest", "api_key": "sk-test"}, + "https://api.typesafe.ai/v1/systemone", + { + "model": "jev-latest", + "answers": {"reachable": {"type": "noul", "noul": 1.0}}, + "usage": {"input_tokens": 12, "output_tokens": 1}, + }, +) +_CLAIRIC_TEST_CONNECTION: Final = ( + { + "model": "cloudflare/clairic", + "api_key": "sk-test", + "api_base": "https://api.cloudflare.com/client/v4/accounts/acct/ai/run", + }, + "https://api.cloudflare.com/client/v4/accounts/acct/ai/run/@cf/cloudflare/clairic", + { + "result": { + "model": "clairic", + "answers": {"status": {"type": "text", "value": "ok"}}, + "truncated": False, + "usage": {"input_tokens": 9, "output_tokens": 0}, + }, + "success": True, + }, +) + + +@pytest.mark.parametrize( + ("mode", "litellm_params", "upstream_url", "upstream_json"), + ( + ("decisions", *_TYPESAFE_TEST_CONNECTION), + ("evaluation", *_TYPESAFE_TEST_CONNECTION), + ("extraction", *_CLAIRIC_TEST_CONNECTION), + ), + ids=("decisions", "legacy-evaluation", "extraction"), +) +def test_test_model_connection_accepts_decision_and_extraction_modes( + monkeypatch, + mode: Literal["evaluation", "decisions", "extraction"], + litellm_params: Mapping[str, str], + upstream_url: str, + upstream_json: Mapping[str, object], ): monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) litellm.in_memory_llm_clients_cache.flush_cache() @@ -5315,20 +5355,8 @@ def test_test_model_connection_accepts_decision_modes( ), respx.mock(assert_all_called=True) as respx_mock, ): - upstream = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond( - json={ - "model": "jev-latest", - "answers": {"reachable": {"type": "noul", "noul": 1.0}}, - "usage": {"input_tokens": 12, "output_tokens": 1}, - } - ) - response = client.post( - "/health/test_connection", - json={ - "mode": mode, - "litellm_params": {"model": "typesafe/jev-latest", "api_key": "sk-test"}, - }, - ) + upstream = respx_mock.post(upstream_url).respond(json=upstream_json) + response = client.post("/health/test_connection", json={"mode": mode, "litellm_params": litellm_params}) assert response.status_code == 200, response.text assert response.json()["status"] == "success" diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 7af747fc26d..8a74f2d50b4 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -48,6 +48,7 @@ from litellm.proxy.spend_tracking.ptu_feature_flag import PTU_COST_ATTRIBUTION_E from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token from litellm.router_utils.ptu_shares import PTUTeamCeiling from litellm.types.caching import RedisPipelineIncrementOperation +from litellm.types.extraction import ExtractionResponse from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.mcp import MCPPreCallRequestObject, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -8686,3 +8687,78 @@ async def test_a2a_url_target_owns_invocation_fee_and_request_limit( await _rpm_request(limiter, cache, auth, "a2a/cheap") assert denied.value.status_code == 429 assert "expensive" in str(denied.value.detail) + + +def _extraction_request(state: str, call_id: str) -> dict[str, object]: + return { + "model": "invoice-reader", + "state": state, + "fields": {"vendor": {"type": "text", "instructions": "Who sent the invoice"}}, + "litellm_call_id": call_id, + } + + +@pytest.mark.asyncio +async def test_an_extraction_request_passes_a_tpm_limited_key_without_an_injected_output_cap(): + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + key: Final = UserAPIKeyAuth(api_key=hash_token("sk-extraction-cap"), tpm_limit=2000) + data: Final = _extraction_request("Invoice from Acme Tools", "extraction-cap") + sent: Final = dict(data) + + await handler.async_pre_call_hook(user_api_key_dict=key, cache=cache, data=data, call_type="aextraction") + + assert data == sent + + +@pytest.mark.asyncio +async def test_an_extraction_request_reserves_its_state_and_no_output_budget(monkeypatch): + monkeypatch.delenv("LITELLM_TPM_TOKEN_RESERVATION_ENABLED", raising=False) + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + key: Final = UserAPIKeyAuth(api_key=hash_token("sk-extraction-estimate"), tpm_limit=1_000_000) + state_tokens: Final = 500 + + reserved: Final = await _reserved_tokens_for( + handler, cache, key, _extraction_request("word" * state_tokens, "extraction-estimate"), call_type="aextraction" + ) + + assert state_tokens <= reserved < 2 * state_tokens + + +@pytest.mark.asyncio +async def test_a_successful_extraction_counts_its_input_tokens_toward_the_key_tpm_limit(monkeypatch): + monkeypatch.setattr(_PROXY_MaxParallelRequestsHandler, "get_rate_limit_type", lambda self: "total") + cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler(internal_usage_cache=InternalUsageCache(cache)) + api_key: Final = hash_token("sk-extraction-usage") + key: Final = UserAPIKeyAuth(api_key=api_key, tpm_limit=500) + await handler.async_pre_call_hook( + user_api_key_dict=key, cache=cache, data=_extraction_request("Invoice", "extraction-1"), call_type="aextraction" + ) + + await handler.async_log_success_event( + kwargs={ + "litellm_call_id": "extraction-1", + "standard_logging_object": {"metadata": {"user_api_key_hash": api_key}}, + }, + response_obj=ExtractionResponse.model_validate( + { + "model": "invoice-reader", + "answers": {"vendor": {"type": "text", "value": "Acme Tools"}}, + "truncated": False, + "usage": {"input_tokens": 480, "output_tokens": 0}, + } + ), + start_time=datetime.now(), + end_time=datetime.now(), + ) + + with pytest.raises(HTTPException) as refused: + await handler.async_pre_call_hook( + user_api_key_dict=key, + cache=cache, + data=_extraction_request("Invoice " * 40, "extraction-2"), + call_type="aextraction", + ) + assert refused.value.status_code == 429 diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index 39423939b68..d742956bb59 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -3551,6 +3551,30 @@ class TestOverrideOpenAIResponseModel: assert response_obj.model == "downstream-model" + def test_frozen_response_comes_back_as_a_copy_carrying_the_requested_model(self): + from litellm.types.extraction import ExtractionResponse + + response_obj = ExtractionResponse.model_validate( + { + "model": "cloudflare/@cf/cloudflare/clairic", + "answers": {"vendor": {"type": "text", "value": "Acme Tools"}}, + "truncated": False, + "usage": {"input_tokens": 52, "output_tokens": 0}, + } + ) + response_obj.set_hidden_params({"response_cost": 0.0000052}) + + stamped = _override_openai_response_model( + response_obj=response_obj, + requested_model="invoice-reader", + log_context="test_context", + ) + + assert isinstance(stamped, ExtractionResponse) + assert stamped.model == "invoice-reader" + assert stamped.answers == response_obj.answers + assert stamped.hidden_params["response_cost"] == 0.0000052 + class TestIsAzureModelRouterRequest: """Tests for _is_azure_model_router_request helper""" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 24d3ed678c9..9b6700e9d39 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1073,6 +1073,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "image_edit", "embedding", "evaluation", + "extraction", "guardrail", "image_generation", "video_generation", @@ -9378,7 +9379,7 @@ def test_models_by_provider(): continue elif v["litellm_provider"] == "sagemaker" or v["litellm_provider"] == "bedrock_converse": continue - elif v.get("mode") in ("search", "decisions", "evaluation"): + elif v.get("mode") in ("search", "decisions", "evaluation", "extraction"): continue else: providers.add(v["litellm_provider"]) diff --git a/ui/litellm-dashboard/src/components/AIHub/modelUsageExample.test.ts b/ui/litellm-dashboard/src/components/AIHub/modelUsageExample.test.ts index 25d1b83ebe9..8591e6f82f8 100644 --- a/ui/litellm-dashboard/src/components/AIHub/modelUsageExample.test.ts +++ b/ui/litellm-dashboard/src/components/AIHub/modelUsageExample.test.ts @@ -21,6 +21,16 @@ describe("modelUsageExample", () => { ); }); + it("posts an extraction model to /v1/extraction with typed fields", () => { + const example = modelUsageExample("extraction", BASE_URL, "clairic", "X-Litellm-Key"); + expect(example).toContain('"https://proxy.example.com/v1/extraction"'); + expect(example).toContain('headers={"X-Litellm-Key": "Bearer your_api_key"}'); + expect(example).toContain('"model": "clairic"'); + expect(example).toContain('"fields": {'); + expect(example).not.toContain("chat.completions"); + expect(example).not.toContain("/v1/systemone"); + }); + it("keeps the chat completions example for every other mode", () => { for (const mode of ["chat", null, undefined]) { const example = modelUsageExample(mode, BASE_URL, "gpt-5.5", "Authorization"); @@ -28,6 +38,7 @@ describe("modelUsageExample", () => { expect(example).toContain('base_url="https://proxy.example.com"'); expect(example).toContain('model="gpt-5.5"'); expect(example).not.toContain("/v1/systemone"); + expect(example).not.toContain("/v1/extraction"); } }); diff --git a/ui/litellm-dashboard/src/components/AIHub/modelUsageExample.ts b/ui/litellm-dashboard/src/components/AIHub/modelUsageExample.ts index 2426ec09942..83f2af1e8c6 100644 --- a/ui/litellm-dashboard/src/components/AIHub/modelUsageExample.ts +++ b/ui/litellm-dashboard/src/components/AIHub/modelUsageExample.ts @@ -38,10 +38,33 @@ response = requests.post( print(response.json()["answers"])`; +const extractionExample = (baseUrl: string, model: string, keyHeaderName: string): string => `import requests + +response = requests.post( + ${JSON.stringify(`${baseUrl}/v1/extraction`)}, # Your LiteLLM Proxy URL + headers={${JSON.stringify(keyHeaderName)}: "Bearer your_api_key"}, + json={ + "model": ${JSON.stringify(model)}, + "state": "Invoice #4471 from Acme Tools. Total $174.00, due in 30 days.", + "fields": { + "vendor": {"type": "text", "instructions": "Who sent the invoice?"}, + "total": {"type": "number", "unit": "USD"} + } + } +) + +print(response.json()["answers"])`; + export const modelUsageExample = ( mode: string | null | undefined, baseUrl: string, model: string, keyHeaderName: string, -): string => - isDecisionMode(mode) ? systemOneExample(baseUrl, model, keyHeaderName) : chatCompletionsExample(baseUrl, model); +): string => { + if (mode === "extraction") { + return extractionExample(baseUrl, model, keyHeaderName); + } + return isDecisionMode(mode) + ? systemOneExample(baseUrl, model, keyHeaderName) + : chatCompletionsExample(baseUrl, model); +}; diff --git a/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx b/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx index cdc681a1c91..a95d9ffbfb8 100644 --- a/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/add_model/AddModelForm.integration.test.tsx @@ -465,7 +465,7 @@ describe("AddModelForm", () => { expect(await screen.findByRole("button", { name: "Add Model" })).toBeInTheDocument(); }); - it("offers the Decisions mode", async () => { + it.each(["Decisions - /v1/decisions", "Extraction - /v1/extraction"])("offers the %s mode", async (option) => { const mockUseAuthorized = vi.mocked(await import("@/app/(dashboard)/hooks/useAuthorized")); mockUseAuthorized.default.mockReturnValue(mockAuthorizedUser("proxy_admin", "user-1", true)); @@ -474,7 +474,7 @@ describe("AddModelForm", () => { await screen.findByText("Provider"); await userEvent.click(screen.getByRole("combobox", { name: "Mode" })); - expect(await screen.findByRole("option", { name: "Decisions - /v1/decisions", exact: true })).toBeInTheDocument(); + expect(await screen.findByRole("option", { name: option, exact: true })).toBeInTheDocument(); }); it("shows only the Close button in the connection test dialog footer", async () => { diff --git a/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx b/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx index 45ffe969a21..63def76db8f 100644 --- a/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx +++ b/ui/litellm-dashboard/src/components/add_model/add_model_modes.tsx @@ -14,6 +14,7 @@ export const TEST_MODES = [ { value: "anthropic_messages", label: "Anthropic Messages - /v1/messages" }, { value: "ocr", label: "OCR - /ocr" }, { value: "decisions", label: "Decisions - /v1/decisions" }, + { value: "extraction", label: "Extraction - /v1/extraction" }, ]; // Define the available auto router routing strategies diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 4bd945b13ca..244141ec076 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -5333,6 +5333,39 @@ export interface paths { patch: operations["assemblyai_proxy_route_eu_assemblyai__endpoint__patch"]; trace?: never; }; + "/extraction": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Extract typed fields from a state + * @description Read a `state` text and up to 4 images and get one typed answer per named field in `fields`. + * + * Field types are `text`, `number`, `date`, `time`, `datetime`, `duration`, `list` (set `items`), `estimate` + * (set `min` and `max`) and `forecast` (set `min`, `max` and `horizon`). Each answer carries its `value`, which is + * null when the input does not determine it, plus `confidence`, `p_null`, text `span`s into the state and, for + * estimates, `mean`, `median`, `p10` and `p90`. Images are data URLs or `{content_type, base64}` objects, remote + * URLs are not accepted. An unknown field type, a field name that is not 1 to 64 letters, digits, `_`, `.` or `-` + * starting with a letter or `_`, a remote image URL or more than 4 images is rejected with a 400 before any + * provider is called. Any other top-level key, `stream: true` and OpenAI generation settings such as `temperature` + * included, is rejected with a 400 naming it unless `drop_params` or the deployment's `additional_drop_params` + * drops it. Settings inside `fields` and `options` go to the provider as sent, so a field missing a setting its + * type needs or an empty `state` with no images comes back with the provider's own error. `model` is any + * extraction model in the proxy model list, such as Cloudflare Clairic. The request and the answers keep the + * provider's format. + */ + post: operations["extraction_extraction_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/fal_ai/{endpoint}": { parameters: { query?: never; @@ -20204,6 +20237,39 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/extraction": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Extract typed fields from a state + * @description Read a `state` text and up to 4 images and get one typed answer per named field in `fields`. + * + * Field types are `text`, `number`, `date`, `time`, `datetime`, `duration`, `list` (set `items`), `estimate` + * (set `min` and `max`) and `forecast` (set `min`, `max` and `horizon`). Each answer carries its `value`, which is + * null when the input does not determine it, plus `confidence`, `p_null`, text `span`s into the state and, for + * estimates, `mean`, `median`, `p10` and `p90`. Images are data URLs or `{content_type, base64}` objects, remote + * URLs are not accepted. An unknown field type, a field name that is not 1 to 64 letters, digits, `_`, `.` or `-` + * starting with a letter or `_`, a remote image URL or more than 4 images is rejected with a 400 before any + * provider is called. Any other top-level key, `stream: true` and OpenAI generation settings such as `temperature` + * included, is rejected with a 400 naming it unless `drop_params` or the deployment's `additional_drop_params` + * drops it. Settings inside `fields` and `options` go to the provider as sent, so a field missing a setting its + * type needs or an empty `state` with no images comes back with the provider's own error. `model` is any + * extraction model in the proxy model list, such as Cloudflare Clairic. The request and the answers keep the + * provider's format. + */ + post: operations["extraction_v1_extraction_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/files": { parameters: { query?: never; @@ -27515,7 +27581,7 @@ export interface components { * Mode * @description The mode to test the model with. If not provided, resolved the way /health does: the deployment's model_info.mode (only while the request tests the deployment's own model), then the mode the provider requires for that model, then the model cost map. */ - mode?: ("chat" | "completion" | "embedding" | "audio_speech" | "audio_transcription" | "image_generation" | "image_edit" | "video_generation" | "batch" | "rerank" | "realtime" | "responses" | "anthropic_messages" | "ocr" | "evaluation" | "decisions") | null; + mode?: ("chat" | "completion" | "embedding" | "audio_speech" | "audio_transcription" | "image_generation" | "image_edit" | "video_generation" | "batch" | "rerank" | "realtime" | "responses" | "anthropic_messages" | "ocr" | "evaluation" | "decisions" | "extraction") | null; /** * Model Info * @description Model info for the health check @@ -28399,7 +28465,7 @@ export interface components { * CallTypes * @enum {string} */ - CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; + CallTypes: "embedding" | "aembedding" | "completion" | "acompletion" | "atext_completion" | "text_completion" | "image_generation" | "aimage_generation" | "image_edit" | "aimage_edit" | "moderation" | "amoderation" | "atranscription" | "transcription" | "aspeech" | "speech" | "rerank" | "arerank" | "search" | "asearch" | "decisions" | "adecisions" | "systemone" | "asystemone" | "extraction" | "aextraction" | "_arealtime" | "_aresponses_websocket" | "create_batch" | "acreate_batch" | "aretrieve_batch" | "retrieve_batch" | "acancel_batch" | "cancel_batch" | "pass_through_endpoint" | "anthropic_messages" | "aanthropic_messages" | "get_assistants" | "aget_assistants" | "create_assistants" | "acreate_assistants" | "delete_assistant" | "adelete_assistant" | "acreate_thread" | "create_thread" | "aget_thread" | "get_thread" | "a_add_message" | "add_message" | "aget_messages" | "get_messages" | "arun_thread" | "run_thread" | "arun_thread_stream" | "run_thread_stream" | "afile_retrieve" | "file_retrieve" | "afile_delete" | "file_delete" | "afile_list" | "file_list" | "acreate_file" | "create_file" | "afile_content" | "file_content" | "create_fine_tuning_job" | "acreate_fine_tuning_job" | "create_video" | "acreate_video" | "video_generation" | "avideo_generation" | "avideo_retrieve" | "video_retrieve" | "avideo_content" | "video_content" | "video_remix" | "avideo_remix" | "video_list" | "avideo_list" | "video_retrieve_job" | "avideo_retrieve_job" | "video_delete" | "avideo_delete" | "video_create_character" | "avideo_create_character" | "video_get_character" | "avideo_get_character" | "video_edit" | "avideo_edit" | "video_extension" | "avideo_extension" | "vector_store_file_create" | "avector_store_file_create" | "vector_store_file_list" | "avector_store_file_list" | "vector_store_file_retrieve" | "avector_store_file_retrieve" | "vector_store_file_content" | "avector_store_file_content" | "vector_store_file_update" | "avector_store_file_update" | "vector_store_file_delete" | "avector_store_file_delete" | "vector_store_create" | "avector_store_create" | "vector_store_search" | "avector_store_search" | "ingest" | "aingest" | "query" | "aquery" | "create_interaction" | "acreate_interaction" | "create_container" | "acreate_container" | "list_containers" | "alist_containers" | "retrieve_container" | "aretrieve_container" | "delete_container" | "adelete_container" | "list_container_files" | "alist_container_files" | "upload_container_file" | "aupload_container_file" | "create_sandbox" | "acreate_sandbox" | "delete_sandbox" | "adelete_sandbox" | "run_code" | "arun_code" | "code_interpreter_tool" | "acode_interpreter_tool" | "acancel_fine_tuning_job" | "cancel_fine_tuning_job" | "alist_fine_tuning_jobs" | "list_fine_tuning_jobs" | "aretrieve_fine_tuning_job" | "retrieve_fine_tuning_job" | "responses" | "aresponses" | "alist_input_items" | "llm_passthrough_route" | "allm_passthrough_route" | "generate_content" | "agenerate_content" | "generate_content_stream" | "agenerate_content_stream" | "ocr" | "aocr" | "call_mcp_tool" | "list_mcp_tools" | "asend_message" | "send_message" | "acreate_skill"; /** CallbackDelete */ CallbackDelete: { /** Callback Name */ @@ -32203,6 +32269,69 @@ export interface components { * @enum {string} */ ExportType: "daily" | "daily_with_keys" | "daily_with_models" | "daily_with_users"; + /** ExtractionAnswer */ + ExtractionAnswer: { + /** Confidence */ + confidence?: number | null; + guess?: components["schemas"]["JsonValue"]; + mean?: components["schemas"]["JsonValue"]; + median?: components["schemas"]["JsonValue"]; + p10?: components["schemas"]["JsonValue"]; + p90?: components["schemas"]["JsonValue"]; + /** P Above */ + p_above?: number | null; + /** P Below */ + p_below?: number | null; + /** P Null */ + p_null?: number | null; + /** Quantiles */ + quantiles?: components["schemas"]["JsonValue"][] | null; + /** Span */ + span?: number[] | null; + /** Spans */ + spans?: number[][] | null; + /** Steps */ + steps?: components["schemas"]["JsonValue"][] | null; + /** Type */ + type: string; + /** @description The answer, null when the input does not determine it */ + value: components["schemas"]["JsonValue"]; + } & { + [key: string]: unknown; + }; + /** ExtractionResponse */ + ExtractionResponse: { + /** Answers */ + answers: { + [key: string]: components["schemas"]["ExtractionAnswer"]; + }; + /** Model */ + model: string; + /** State Text */ + state_text?: string | null; + /** Truncated */ + truncated: boolean; + usage: components["schemas"]["ExtractionUsage"]; + } & { + [key: string]: unknown; + }; + /** ExtractionUsage */ + ExtractionUsage: { + /** Image Tokens */ + image_tokens?: number | null; + /** Input Tokens */ + input_tokens: number; + /** Output Tokens */ + output_tokens: number; + /** Passes */ + passes?: number | null; + /** Schema Tokens */ + schema_tokens?: number | null; + /** State Tokens */ + state_tokens?: number | null; + } & { + [key: string]: unknown; + }; /** * FacetListResponse * @description The distinct values one column takes over a filtered query. `data` holds bare values, not entity rows. @@ -59011,6 +59140,180 @@ export interface operations { }; }; }; + extraction_extraction_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "fields": { + * "days_to_pay": { + * "max": 90, + * "min": 0, + * "type": "estimate" + * }, + * "invoice_date": { + * "type": "date" + * }, + * "items": { + * "instructions": "What was bought?", + * "items": "text", + * "type": "list" + * }, + * "total": { + * "type": "number", + * "unit": "USD" + * }, + * "vendor": { + * "instructions": "Who sent the invoice?", + * "type": "text" + * } + * }, + * "model": "clairic", + * "state": "Invoice #4471 from Acme Tools, dated 2026-03-03. 12 hammers. Total $174. Payment due in 30 days." + * } + */ + "application/json": { + /** + * Fields + * @description Field name to typed field spec. Each name becomes a key in the answers + */ + fields: { + [key: string]: { + /** + * Decimals + * @description Numbers, durations and number lists only, 0 to 6 + */ + decimals?: number | null; + /** + * Horizon + * @description Forecast only, number of steps + */ + horizon?: number | null; + /** + * Instructions + * @description What the field should contain + */ + instructions?: string | null; + /** + * Integer + * @description Numbers and durations only + */ + integer?: boolean | null; + /** Interval */ + interval?: ("step" | "minute" | "hour" | "day" | "week" | "month" | "quarter" | "year") | null; + /** + * Items + * @description Item type, required for list fields + */ + items?: ("text" | "number" | "date") | null; + /** + * Max + * @description Estimate and forecast upper bound + */ + max?: number | string | null; + /** + * Min + * @description Estimate and forecast lower bound + */ + min?: number | string | null; + /** + * Null Threshold + * @description Probability of null above which the value is null + */ + null_threshold?: number | null; + /** Scale */ + scale?: ("linear" | "log") | null; + /** + * Step + * @description Estimate grid step, or month or year + */ + step?: number | string | null; + /** + * Type + * @description The answer type for this field + * @enum {string} + */ + type: "text" | "number" | "date" | "time" | "datetime" | "duration" | "list" | "estimate" | "forecast"; + /** + * Unit + * @description Unit label. For durations, seconds through years + */ + unit?: string | null; + } & { + [key: string]: unknown; + }; + }; + /** + * Images + * @description Up to 4 images as data URLs or content_type plus base64, no remote URLs + */ + images?: (string | ({ + /** + * Base64 + * @description Base64-encoded image bytes + */ + base64: string; + /** + * Content Type + * @description The image media type, such as image/png + */ + content_type: string; + } & { + [key: string]: unknown; + }))[] | null; + /** + * Model + * @description An extraction model from the proxy model_list + */ + model: string; + options?: ({ + /** + * Include Quantiles + * @description Add 99 percentiles to estimates + */ + include_quantiles?: boolean | null; + /** + * Return State Text + * @description Return the text that spans index into + */ + return_state_text?: boolean | null; + } & { + [key: string]: unknown; + }) | null; + /** + * Reference Date + * @description ISO date that anchors dates written without a year + */ + reference_date?: string | null; + /** + * State + * @description The text to read. Must be non-empty unless images are sent. Text answers are spans of it + * @default + */ + state?: string; + } & { + [key: string]: unknown; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ExtractionResponse"]; + }; + }; + }; + }; fal_ai_proxy_route_fal_ai__endpoint__get: { parameters: { query?: never; @@ -78336,6 +78639,180 @@ export interface operations { }; }; }; + extraction_v1_extraction_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + /** + * @example { + * "fields": { + * "days_to_pay": { + * "max": 90, + * "min": 0, + * "type": "estimate" + * }, + * "invoice_date": { + * "type": "date" + * }, + * "items": { + * "instructions": "What was bought?", + * "items": "text", + * "type": "list" + * }, + * "total": { + * "type": "number", + * "unit": "USD" + * }, + * "vendor": { + * "instructions": "Who sent the invoice?", + * "type": "text" + * } + * }, + * "model": "clairic", + * "state": "Invoice #4471 from Acme Tools, dated 2026-03-03. 12 hammers. Total $174. Payment due in 30 days." + * } + */ + "application/json": { + /** + * Fields + * @description Field name to typed field spec. Each name becomes a key in the answers + */ + fields: { + [key: string]: { + /** + * Decimals + * @description Numbers, durations and number lists only, 0 to 6 + */ + decimals?: number | null; + /** + * Horizon + * @description Forecast only, number of steps + */ + horizon?: number | null; + /** + * Instructions + * @description What the field should contain + */ + instructions?: string | null; + /** + * Integer + * @description Numbers and durations only + */ + integer?: boolean | null; + /** Interval */ + interval?: ("step" | "minute" | "hour" | "day" | "week" | "month" | "quarter" | "year") | null; + /** + * Items + * @description Item type, required for list fields + */ + items?: ("text" | "number" | "date") | null; + /** + * Max + * @description Estimate and forecast upper bound + */ + max?: number | string | null; + /** + * Min + * @description Estimate and forecast lower bound + */ + min?: number | string | null; + /** + * Null Threshold + * @description Probability of null above which the value is null + */ + null_threshold?: number | null; + /** Scale */ + scale?: ("linear" | "log") | null; + /** + * Step + * @description Estimate grid step, or month or year + */ + step?: number | string | null; + /** + * Type + * @description The answer type for this field + * @enum {string} + */ + type: "text" | "number" | "date" | "time" | "datetime" | "duration" | "list" | "estimate" | "forecast"; + /** + * Unit + * @description Unit label. For durations, seconds through years + */ + unit?: string | null; + } & { + [key: string]: unknown; + }; + }; + /** + * Images + * @description Up to 4 images as data URLs or content_type plus base64, no remote URLs + */ + images?: (string | ({ + /** + * Base64 + * @description Base64-encoded image bytes + */ + base64: string; + /** + * Content Type + * @description The image media type, such as image/png + */ + content_type: string; + } & { + [key: string]: unknown; + }))[] | null; + /** + * Model + * @description An extraction model from the proxy model_list + */ + model: string; + options?: ({ + /** + * Include Quantiles + * @description Add 99 percentiles to estimates + */ + include_quantiles?: boolean | null; + /** + * Return State Text + * @description Return the text that spans index into + */ + return_state_text?: boolean | null; + } & { + [key: string]: unknown; + }) | null; + /** + * Reference Date + * @description ISO date that anchors dates written without a year + */ + reference_date?: string | null; + /** + * State + * @description The text to read. Must be non-empty unless images are sent. Text answers are spans of it + * @default + */ + state?: string; + } & { + [key: string]: unknown; + }; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ExtractionResponse"]; + }; + }; + }; + }; list_files_v1_files_get: { parameters: { query?: { From 4aec80f5ee1b3bbd2804abc123fe6663be77f775 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 20 Oct 2026 16:00:00 -0700 Subject: [PATCH 017/235] fix(mcp): upgrade to MCP SDK 2.3.0 and raise the per-event SSE size cap (#47034) * fix(mcp): upgrade to MCP SDK 2.3.0 and raise the per-event SSE size cap MCP SDK 2.x reads Streamable HTTP responses through httpx2's SSE parser, which rejects any single event over 1 MiB, so a large tools/call result failed with 'SSE stream ended without a response'. mcp 2.3.0 exposes max_sse_event_size, and MCPClient now passes a 64 MiB default, configurable with LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(mcp): fall back for invalid SSE event limits * test(mcp): verify configured SSE limits and recovery * chore(mcp): remove redundant dependency comment --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/constants.py | 3 + litellm/experimental_mcp_client/client.py | 2 + pyproject.toml | 11 +-- tests/integration/_support/mcp.py | 6 +- tests/integration/mcp/test_mcp_transports.py | 72 +++++++++++++++++++ .../test_mcp_client.py | 4 +- tests/unit/test_constants.py | 28 ++++++++ uv.lock | 25 +++---- 8 files changed, 131 insertions(+), 20 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 70be1e56341..5b99a0f73dd 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -258,6 +258,9 @@ MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS: Final = get_env_int("MCP_PER_USER_TOKE # MCP timeout defaults (seconds). Override via env vars for slow/custom MCP servers. MCP_CLIENT_TIMEOUT: Final = get_env_float("LITELLM_MCP_CLIENT_TIMEOUT", 60.0) +MCP_MAX_SSE_EVENT_SIZE_BYTES: Final = get_env_int_in_range( + "LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES", 64 * 1024 * 1024, 1, sys.maxsize +) MCP_TOOL_LISTING_TIMEOUT: Final = get_env_float("LITELLM_MCP_TOOL_LISTING_TIMEOUT", 30.0) MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES: Final = get_env_int("LITELLM_MCP_UPSTREAM_RATE_LIMIT_MAX_RETRIES", 3) MCP_UPSTREAM_RATE_LIMIT_BASE_DELAY_SECONDS: Final = get_env_float( diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 85075b52b71..79887e4ab99 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -95,6 +95,7 @@ from pydantic import AnyUrl, TypeAdapter from litellm._logging import verbose_logger from litellm.constants import ( MCP_CLIENT_TIMEOUT, + MCP_MAX_SSE_EVENT_SIZE_BYTES, MCP_NPM_CACHE_DIR, MCP_TOOL_LISTING_MAX_PAGES, MCP_TOOL_LISTING_TIMEOUT, @@ -647,6 +648,7 @@ class MCPClient: transport_ctx: Final = streamable_http_client( url=self.server_url, http_client=http_client, + max_sse_event_size=MCP_MAX_SSE_EVENT_SIZE_BYTES, ) return transport_ctx, http_client diff --git a/pyproject.toml b/pyproject.toml index 4cb529bbdb3..a11708b15f2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,8 +75,8 @@ proxy = [ "boto3>=1.43.1,<2.0", "azure-identity>=1.25.2,<2.0", "azure-storage-blob>=12.28.0,<13.0", - "mcp>=2.2.0,<3", - "httpx2>=2.5.0,<3", + "mcp>=2.3.0,<3", + "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3", "litellm-proxy-extras==0.4.111", "litellm-enterprise==0.1.79", @@ -125,7 +125,7 @@ utils = [ "numpydoc>=1.8.0,<2.0", ] caching = ["diskcache>=5.6.3,<6.0"] -mcp = ["mcp>=2.2.0,<3", "httpx2>=2.5.0,<3", "pydantic>=2.12.0,<3"] +mcp = ["mcp>=2.3.0,<3", "httpx2>=2.10.0,<3", "pydantic>=2.12.0,<3"] # Driver for the MongoDB Atlas vector store; Atlas Vector Search has no HTTP query API. # The floor is 4.9 because that is the release AsyncMongoClient landed in. # SAML SSO for the admin UI. python3-saml pulls in xmlsec/lxml, whose wheels @@ -243,7 +243,7 @@ e2e-dev = [ "locust==2.45.0", "anthropic==0.84.0", "psutil==7.2.2", - "mcp>=2.2.0,<3", + "mcp>=2.3.0,<3", ] admin-mcp = [ "litellm-admin-mcp @ https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz#sha256=86b930f6706fb2da10d53d1798ee7fd14e44fb0afdef0da122cf0e6bb88fa2c3 ; python_version >= '3.12'", @@ -303,7 +303,7 @@ healthcheck = [ benchmarks = [ "pytest==9.0.3", "pytest-codspeed==4.3.0", - "mcp>=2.2.0,<3", + "mcp>=2.3.0,<3", "a2a-sdk==1.1.0", ] @@ -363,6 +363,7 @@ override-dependencies = [ # a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0. "packaging>=24.0", "cryptography>=50.0.0,<51.0", + "mcp>=2.3.0,<3", ] default-groups = ["dev"] required-version = ">=0.10.9" diff --git a/tests/integration/_support/mcp.py b/tests/integration/_support/mcp.py index f69079ce90b..99bf71cd459 100644 --- a/tests/integration/_support/mcp.py +++ b/tests/integration/_support/mcp.py @@ -282,6 +282,7 @@ class ScriptedTool: respond: Callable[[JsonRpc], Reply | JsonRpc] description: str | Callable[[Mapping[str, str]], str] | None = None input_schema: JsonRpc = field(default_factory=lambda: {"type": "object"}) + event_stream: bool = False def listing(self, headers: Mapping[str, str]) -> JsonRpc: described: Final = self.description(headers) if callable(self.description) else self.description @@ -334,7 +335,10 @@ def scripted_peer(*tools: ScriptedTool) -> Iterator[McpPeer]: if tool is None: return jsonrpc_error(identity, -32602, "unknown tool") produced: Final = tool.respond(body["params"]) - return produced if isinstance(produced, Reply) else jsonrpc_reply(identity, produced) + reply: Final = produced if isinstance(produced, Reply) else jsonrpc_reply(identity, produced) + if not tool.event_stream: + return reply + return Reply(content_type="text/event-stream", body=b"event: message\ndata: " + reply.body + b"\n\n") with wire_server(provider) as wire: yield McpPeer(wire.url + "/mcp", observed) diff --git a/tests/integration/mcp/test_mcp_transports.py b/tests/integration/mcp/test_mcp_transports.py index 928aa9ff121..ef240cba570 100644 --- a/tests/integration/mcp/test_mcp_transports.py +++ b/tests/integration/mcp/test_mcp_transports.py @@ -2,10 +2,12 @@ import builtins import json import uuid from collections.abc import Mapping +from pathlib import Path from typing import Final import httpx import pytest +import yaml from mcp.shared.exceptions import MCPError from integration._support.client import Gateway from integration._support.database import read_rows @@ -17,15 +19,21 @@ from integration._support.mcp import ( McpCaller, Outcome, PeerKind, + ScriptedTool, _outcome_from_rpc, _parse_rpc_body, + call_tool, + listed_tools, mcp_peer, official_client_outcomes, peer_of, register_mcp, + scripted_peer, + text_result, tool_calls, ) from integration._support.mcp_grants import create_toolset +from integration._support.process import owned_proxy ADD: Final = {"http": "add", "sse": "add", "stdio": "add", "openapi": "getpet"} ARGUMENTS: Final = {"add": {"a": 3, "b": 4}, "getpet": {"petId": "7"}} @@ -817,3 +825,67 @@ def test_sdk_progress_and_cancellation_reach_upstream_without_losing_session( ), ) as gateway: asyncio.run(exercise(gateway)) + + +def test_tool_result_larger_than_one_mib_in_a_single_sse_event_reaches_the_caller(gateway: Gateway) -> None: + payload: Final = "x" * (3 * 1024 * 1024) + tool: Final = ScriptedTool("dump", lambda _: text_result(payload), event_stream=True) + + with scripted_peer(tool) as peer, gateway.scenario() as scenario: + alias: Final = "bigsse" + uuid.uuid4().hex[:8] + identity: Final = register_mcp(scenario, peer, alias) + key: Final = scenario.key(object_permission={"mcp_servers": [identity]}) + name: Final = next(full for full in listed_tools(gateway, key, identity) if full.endswith("dump")) + response: Final = call_tool(gateway, key, identity, name, {}) + assert response.status_code == 200, response.text[:500] + body: Final = response.json() + assert body["isError"] is False, response.text[:500] + assert body["content"][0]["text"] == payload + + +def test_configured_sse_limit_rejects_large_results_and_allows_the_next_call(tmp_path: Path) -> None: + limit: Final = 4096 + payload: Final = "x" * 512 + small: Final = ScriptedTool("small", lambda _: text_result(payload), event_stream=True) + large: Final = ScriptedTool("large", lambda _: text_result("x" * (2 * limit)), event_stream=True) + key: Final = "sk-local-mcp-sse-limit-regression-key" + with scripted_peer(small, large) as peer, httpx.Client() as client: + config: Final = tmp_path / "sse-limit.yaml" + config.write_text( + yaml.safe_dump( + { + "general_settings": {"master_key": key, "store_model_in_db": False}, + "mcp_servers": {"limited": peer.registration()}, + } + ) + ) + seed: Final = Gateway(client, key, peer.url) + with owned_proxy( + seed, + tmp_path / "proxy", + { + "STORE_MODEL_IN_DB": "False", + "DISABLE_SCHEMA_UPDATE": "true", + "LITELLM_LOCAL_MODEL_COST_MAP": "True", + "LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES": str(limit), + }, + config=config, + database_setup=(), + remove_environment=( + "DATABASE_URL", + "DATABASE_URL_READ_REPLICA", + "DIRECT_URL", + "INTEGRATION_PROXY_DATABASE_URL", + "INTEGRATION_PROXY_READ_REPLICA_URL", + ), + ) as gateway: + assert set(listed_tools(gateway, key)) == {"small", "large"} + for name in ("small", "large", "small"): + response: Final = call_tool(gateway, key, "limited", name, {}) + assert response.status_code == 200, response.text[:500] + body: Final = response.json() + assert body["isError"] is (name == "large"), response.text[:500] + if name == "large": + assert str(limit) in body["content"][0]["text"], response.text[:500] + else: + assert body["content"][0]["text"] == payload, response.text[:500] diff --git a/tests/unit/experimental_mcp_client/test_mcp_client.py b/tests/unit/experimental_mcp_client/test_mcp_client.py index ba7786f8cd5..b4485bcc5f1 100644 --- a/tests/unit/experimental_mcp_client/test_mcp_client.py +++ b/tests/unit/experimental_mcp_client/test_mcp_client.py @@ -1500,8 +1500,8 @@ def test_mcp_extra_matches_proxy_extra_and_supports_streamable_http(): } specifier: Final = Requirement(mcp_extra["mcp"]).specifier - assert not specifier.contains("1.28.1") - assert specifier.contains("2.2.0") + assert not specifier.contains("2.2.0") + assert specifier.contains("2.3.0") with (pyproject_path.parent / "uv.lock").open("rb") as f: locked = tomllib.load(f) mcp_versions: Final = [package["version"] for package in locked["package"] if package["name"] == "mcp"] diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py index 0d0e77765b7..99b049f1803 100644 --- a/tests/unit/test_constants.py +++ b/tests/unit/test_constants.py @@ -1,6 +1,7 @@ import ast, asyncio, os import inspect import json +import sys from unittest import mock import httpx @@ -117,6 +118,33 @@ def test_cli_jwt_expiration_hours_from_environment( importlib.reload(litellm.constants) +@pytest.mark.parametrize( + ("raw", "expected"), + ( + (None, 64 * 1024 * 1024), + ("", 64 * 1024 * 1024), + ("invalid", 64 * 1024 * 1024), + ("0", 64 * 1024 * 1024), + ("-1", 64 * 1024 * 1024), + ("1", 1), + ("200000", 200000), + (str(sys.maxsize), sys.maxsize), + (str(sys.maxsize + 1), 64 * 1024 * 1024), + ), +) +def test_mcp_sse_event_limit_uses_positive_override_or_default(raw: str | None, expected: int) -> None: + try: + with pytest.MonkeyPatch.context() as environment: + if raw is None: + environment.delenv("LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES", raising=False) + else: + environment.setenv("LITELLM_MCP_MAX_SSE_EVENT_SIZE_BYTES", raw) + importlib.reload(constants) + assert constants.MCP_MAX_SSE_EVENT_SIZE_BYTES == expected + finally: + importlib.reload(constants) + + def _is_blank(value: object) -> bool: return isinstance(value, str) and not value.strip() diff --git a/uv.lock b/uv.lock index 297344d1f7a..afc377eb7e4 100644 --- a/uv.lock +++ b/uv.lock @@ -29,6 +29,7 @@ constraints = [ ] overrides = [ { name = "cryptography", specifier = ">=50.0.0,<51.0" }, + { name = "mcp", specifier = ">=2.3.0,<3" }, { name = "packaging", specifier = ">=24.0" }, ] @@ -4902,8 +4903,8 @@ requires-dist = [ { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, { name = "hiredis", marker = "extra == 'proxy'", specifier = ">=3.0.0,<4.0" }, { name = "httpx", extras = ["http2"], specifier = ">=0.28.0,<1.0" }, - { name = "httpx2", marker = "extra == 'mcp'", specifier = ">=2.5.0,<3" }, - { name = "httpx2", marker = "extra == 'proxy'", specifier = ">=2.5.0,<3" }, + { name = "httpx2", marker = "extra == 'mcp'", specifier = ">=2.10.0,<3" }, + { name = "httpx2", marker = "extra == 'proxy'", specifier = ">=2.10.0,<3" }, { name = "huggingface-hub", specifier = ">=0.34.0,<2.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, @@ -4916,8 +4917,8 @@ requires-dist = [ { name = "litellm-proxy-extras", marker = "extra == 'proxy'", editable = "litellm-proxy-extras" }, { name = "llm-sandbox", marker = "extra == 'proxy-runtime'", specifier = ">=0.3.39,<1.0" }, { name = "mangum", marker = "extra == 'proxy-runtime'", specifier = ">=0.17.0,<1.0" }, - { name = "mcp", marker = "extra == 'mcp'", specifier = ">=2.2.0,<3" }, - { name = "mcp", marker = "extra == 'proxy'", specifier = ">=2.2.0,<3" }, + { name = "mcp", marker = "extra == 'mcp'", specifier = ">=2.3.0,<3" }, + { name = "mcp", marker = "extra == 'proxy'", specifier = ">=2.3.0,<3" }, { name = "mlflow", marker = "extra == 'mlflow'", specifier = ">=3.11.1,<4.0" }, { name = "numpy", marker = "extra == 'stt-nvidia-riva'", specifier = ">=1.26.0" }, { name = "numpydoc", marker = "extra == 'utils'", specifier = ">=1.8.0,<2.0" }, @@ -4977,7 +4978,7 @@ provides-extras = ["proxy", "cli", "extra-proxy", "utils", "caching", "mcp", "sa admin-mcp = [{ name = "litellm-admin-mcp", marker = "python_full_version >= '3.12'", url = "https://github.com/BerriAI/liteadmin-mcp/archive/d35ec9c19c117d4c50cc1eccf6ce1296aac25a1a.tar.gz" }] benchmarks = [ { name = "a2a-sdk", specifier = "==1.1.0" }, - { name = "mcp", specifier = ">=2.2.0,<3" }, + { name = "mcp", specifier = ">=2.3.0,<3" }, { name = "pytest", specifier = "==9.0.3" }, { name = "pytest-codspeed", specifier = "==4.3.0" }, ] @@ -5055,7 +5056,7 @@ dev = [ e2e-dev = [ { name = "anthropic", specifier = "==0.84.0" }, { name = "locust", specifier = "==2.45.0" }, - { name = "mcp", specifier = ">=2.2.0,<3" }, + { name = "mcp", specifier = ">=2.3.0,<3" }, { name = "playwright", specifier = "==1.61.0" }, { name = "psutil", specifier = "==7.2.2" }, { name = "websockets", specifier = ">=15.0.1,<16.0" }, @@ -5570,7 +5571,7 @@ wheels = [ [[package]] name = "mcp" -version = "2.2.0" +version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, @@ -5588,22 +5589,22 @@ dependencies = [ { name = "typing-inspection" }, { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/76/31/ac54fb0fdd5b37de704486e288bba4fbbb463f24cfcfedbede407b854513/mcp-2.2.0.tar.gz", hash = "sha256:2dc37ecb1974becdcebdbf7561e7c15a07dbbf20ba21ba16c3593b3038b3afbd", size = 4084129, upload-time = "2026-09-07T16:06:23.439Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9d/8d/e0d339616f4810e9051d4aba6887afab289ab1f81875fe908b606cdfd0e3/mcp-2.3.0.tar.gz", hash = "sha256:8b147a50441cf059dc88c684e0aeed3687f0aa0f39c6cde7b90330effd2b34d8", size = 4171955, upload-time = "2026-10-02T22:06:56.091Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1b/ff/8e7eade68b8a28f7da0ed1085544341b51f9c935dbf6b95c76b7edfea6a0/mcp-2.2.0-py3-none-any.whl", hash = "sha256:bde982589473a060ae145e3406e9a5333fe538c97229ba841f5a7f92be004f81", size = 365656, upload-time = "2026-09-07T16:06:19.711Z" }, + { url = "https://files.pythonhosted.org/packages/20/d7/4a2878f128df729db3ca04e71bfce26405cc2e0c9ebe19c592feeb4c61b9/mcp-2.3.0-py3-none-any.whl", hash = "sha256:dd0c44c089d16453e8ae31a3877a0054d7a2314caaa81f5e0541b9b1734b2377", size = 368664, upload-time = "2026-10-02T22:06:52.119Z" }, ] [[package]] name = "mcp-types" -version = "2.2.0" +version = "2.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pydantic" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/91/762d7755d971aff8a28d75f7961656148edf27875c8026e6385aaab08ae7/mcp_types-2.2.0.tar.gz", hash = "sha256:d3ed53703ddd10d9c6399f29d322bb66f3f67ab41348ac8556ba23e07fedefad", size = 65892, upload-time = "2026-09-07T16:06:25.187Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/2d/7c251e34207f6c51000312fc8839111ac45cfe02023f90b44e7f1051dd8e/mcp_types-2.3.0.tar.gz", hash = "sha256:d1e46549edb35ee19a94940fcee6d1addd7e589ab7ea92dda83f5d84781fc362", size = 65898, upload-time = "2026-10-02T22:06:57.863Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/8f/d7/6ffba5d8cd5dd9b8a19478875c50e04945314ba5074e84d749283f27f62d/mcp_types-2.2.0-py3-none-any.whl", hash = "sha256:ea476b73ee86709ab5abc9452385ed36cc05907e582355622e294595c9a04f13", size = 69106, upload-time = "2026-09-07T16:06:21.461Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/32583066b155bb5d4507e7887dc88ee23118cc4de8e2a12870c49719e2a8/mcp_types-2.3.0-py3-none-any.whl", hash = "sha256:968efdbdaedfab06adae40d378a34395f1090c5921d4be3c9cde283aaf76d91d", size = 69108, upload-time = "2026-10-02T22:06:54.099Z" }, ] [[package]] From d7c3588efdcdf144850fd3facf9924f58c8f7721 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:03:56 +0000 Subject: [PATCH 018/235] feat(nvidia_riva)!: add riva_offline mode for offline-only ASR models (#41021) * feat(nvidia_riva): add riva_offline mode for offline-only ASR models Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(nvidia_riva): assert offline vs streaming via fake service call tracking Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(ui): regenerate schema.d.ts after user soft_budget docstring removal Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(nvidia_riva): parse riva_offline strings, map oversize offline audio to 413 A multipart riva_offline=false no longer turns offline mode on, a malformed value drops under drop_params and answers 400 without it, and a Riva RESOURCE_EXHAUSTED for a message over the server's max now returns a non-retryable 413 instead of a retried 429 * fix(nvidia_riva): apply the request timeout to offline recognition and keep riva_offline on an api_base override Offline Recognize now runs as a gRPC future waited on with the request timeout, so a stalled Riva server returns a 408 and the call is cancelled. riva_offline is a mode flag, not a credential, so an api_base override no longer clears it. The 413 hint for audio over the gRPC message limit is reworded, and the new tests carry type annotations. * fix(nvidia_riva): wait the request timeout in seconds whatever its form and cap the riva client to versions that import A timeout from os.environ, a multipart form field, or an httpx.Timeout reached the gRPC future as a string or object, so result() raised TypeError and every offline call came back as a false 408. The handler now turns the timeout into float seconds with CompletionTimeout.as_limit, taking the read limit of an httpx.Timeout, for both the offline future and the streaming deadline The stt-nvidia-riva extra caps nvidia-riva-client below 2.17, since every later release either pins an older grpcio or ships stubs that need a newer one than the grpcio==1.78.0 pin, so a fresh install failed at import * fix(nvidia_riva): parse the timeout only where the call uses it and drop the riva client cap A malformed timeout no longer fails streaming calls on clients that take no timeout. On offline calls it is a 400, or the default under drop_params, and an httpx.Timeout with no read limit waits the default instead of forever. The nvidia-riva-client cap made installs next to a newer client unresolvable, so the extra goes back to main's specifier. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../audio_transcription/handler.py | 154 ++++++-- litellm/llms/nvidia_riva/common_utils.py | 15 +- litellm/proxy/auth/auth_utils.py | 1 + .../audio_transcription/test_handler.py | 374 +++++++++++++++++- tests/unit/proxy/auth/test_auth_utils.py | 32 ++ 5 files changed, 545 insertions(+), 31 deletions(-) diff --git a/litellm/llms/nvidia_riva/audio_transcription/handler.py b/litellm/llms/nvidia_riva/audio_transcription/handler.py index 651a2215052..748203b22d9 100644 --- a/litellm/llms/nvidia_riva/audio_transcription/handler.py +++ b/litellm/llms/nvidia_riva/audio_transcription/handler.py @@ -16,7 +16,9 @@ The handler is intentionally a thin orchestration layer: 3. Construct ``riva.client.Auth`` honoring NVCF (function-id metadata + TLS) vs self-hosted (any host:port, optional TLS) modes. 4. Stream the audio through Riva's ``streaming_response_generator`` and - aggregate ``is_final`` results into a single transcript. + aggregate ``is_final`` results into a single transcript, or, when + ``riva_offline`` is set, send the whole buffer through the unary + ``offline_recognize`` RPC for NIMs that only ship offline engines. 5. Return a normalized ``TranscriptionResponse`` with ``duration`` exposed on ``_hidden_params`` so cost calculation works. @@ -26,13 +28,19 @@ without the optional STT extras installed. import asyncio import inspect -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping +from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Protocol +import httpx + +import litellm +from litellm.constants import COMPLETION_HTTP_FALLBACK_SECONDS from litellm.litellm_core_utils.audio_utils.utils import ( get_audio_file_name, process_audio_file, ) +from litellm.litellm_core_utils.completion_timeout import CompletionTimeout from litellm.llms.nvidia_riva.audio_transcription.audio_utils import ( resample_to_riva_pcm, ) @@ -45,6 +53,7 @@ from litellm.llms.nvidia_riva.common_utils import ( NvidiaRivaException, grpc_error_to_litellm_exception, ) +from litellm.secret_managers.main import str_to_bool from litellm.types.utils import FileTypes, TranscriptionResponse from litellm.utils import convert_to_model_response_object @@ -67,10 +76,20 @@ class _RivaAuth(Protocol): """Opaque ``riva.client.Auth`` handle.""" +class _RecognizeFuture(Protocol): + def result(self, timeout: float | None = None) -> object: ... + + def done(self) -> bool: ... + + def cancel(self) -> bool: ... + + class _AsrService(Protocol): @property def streaming_response_generator(self) -> Callable[..., Iterable[object]]: ... + def offline_recognize(self, audio_bytes: bytes, config: object, future: bool) -> _RecognizeFuture: ... + class _EndpointingConfig(Protocol): """Opaque ``EndpointingConfig`` protobuf message.""" @@ -128,6 +147,38 @@ def _audio_encoding(riva_asr_module: _RivaAsrModule) -> _AudioEncoding: return riva_asr_module.AudioEncoding +def _parse_riva_offline(value: object, drop_params: bool) -> bool: + match value: + case None: + return False + case bool(): + return value + case str() if (parsed := str_to_bool(value)) is not None: + return parsed + case _ if drop_params: + return False + case _: + raise NvidiaRivaException( + status_code=400, + message=f"riva_offline must be true or false, got {value!r}. Set drop_params to ignore it.", + ) + + +def _deadline_seconds(timeout: float | str | httpx.Timeout, drop_params: bool) -> float: + try: + limit: Final = CompletionTimeout.as_limit(timeout) + except ValueError: + if drop_params: + return COMPLETION_HTTP_FALLBACK_SECONDS + raise NvidiaRivaException( + status_code=400, + message=f"timeout must be a number of seconds, got {timeout!r}. Set drop_params to ignore it.", + ) from None + if not isinstance(limit, httpx.Timeout): + return limit + return COMPLETION_HTTP_FALLBACK_SECONDS if limit.read is None else limit.read + + class NvidiaRivaAudioTranscription: """Sync + async entry point for Riva ASR.""" @@ -138,7 +189,7 @@ class NvidiaRivaAudioTranscription: optional_params: dict, litellm_params: dict, model_response: TranscriptionResponse, - timeout: float, + timeout: float | str | httpx.Timeout | None, logging_obj: "LiteLLMLoggingObj", api_key: str | None, api_base: str | None, @@ -183,7 +234,7 @@ class NvidiaRivaAudioTranscription: optional_params: dict, litellm_params: dict, model_response: TranscriptionResponse, - timeout: float, + timeout: float | str | httpx.Timeout | None, logging_obj: "LiteLLMLoggingObj", api_key: str | None, api_base: str | None, @@ -213,7 +264,7 @@ class NvidiaRivaAudioTranscription: optional_params: dict, litellm_params: dict, model_response: TranscriptionResponse, - timeout: float, + timeout: float | str | httpx.Timeout | None, logging_obj: "LiteLLMLoggingObj", api_key: str | None, api_base: str | None, @@ -259,6 +310,10 @@ class NvidiaRivaAudioTranscription: response_format: Final = request_payload.get("response_format") or "json" timestamp_granularities: Final = request_payload.get("timestamp_granularities") + drop_params: Final = ( + "drop_params" in litellm_params and litellm_params["drop_params"] is True + ) or litellm.drop_params is True + riva_offline: Final = _parse_riva_offline(optional_params.get("riva_offline"), drop_params=drop_params) riva_module, riva_asr_module = _import_riva() auth_obj: Final = self._construct_auth( @@ -272,10 +327,6 @@ class NvidiaRivaAudioTranscription: riva_asr_module=riva_asr_module, recognition_config_dict=recognition_config_dict, ) - streaming_config: Final[_StreamingRecognitionConfig] = riva_asr_module.StreamingRecognitionConfig( - config=recognition_config, interim_results=False - ) - logging_obj.pre_call( input=None, api_key=api_key, @@ -286,24 +337,31 @@ class NvidiaRivaAudioTranscription: "recognition_config": recognition_config_dict, "nvcf_function_id_set": bool(optional_params.get("nvcf_function_id")), "use_ssl": optional_params.get("use_ssl"), + "riva_offline": riva_offline, }, }, ) try: asr_service: Final[_AsrService] = riva_module.ASRService(auth_obj) - audio_chunks: Final = self._iter_audio_chunks(resampled.pcm_bytes) - stream_kwargs: Final[dict[str, object]] = { - "audio_chunks": audio_chunks, - "streaming_config": streaming_config, - } - # Forward the deadline so the stream cannot block forever if the - # server stalls. Older riva-client versions do not accept a - # ``timeout`` kwarg, so pass it only when supported. - if timeout is not None and self._supports_timeout_kwarg(asr_service.streaming_response_generator): - stream_kwargs["timeout"] = float(timeout) - stream: Final = asr_service.streaming_response_generator(**stream_kwargs) - final_results: Final = self._collect_final_results(stream) + responses: Final = ( + self._offline_responses( + asr_service=asr_service, + pcm_bytes=resampled.pcm_bytes, + recognition_config=recognition_config, + timeout=None if timeout is None else _deadline_seconds(timeout, drop_params), + ) + if riva_offline + else self._streaming_responses( + asr_service=asr_service, + riva_asr_module=riva_asr_module, + pcm_bytes=resampled.pcm_bytes, + recognition_config=recognition_config, + timeout=timeout, + drop_params=drop_params, + ) + ) + final_results: Final = self._collect_final_results(responses, require_final=not riva_offline) except NvidiaRivaException: raise except Exception as e: @@ -409,6 +467,54 @@ class NvidiaRivaAudioTranscription: return config + def _streaming_responses( + self, + asr_service: _AsrService, + riva_asr_module: _RivaAsrModule, + pcm_bytes: bytes, + recognition_config: _RecognitionConfig, + timeout: float | str | httpx.Timeout | None, + drop_params: bool, + ) -> Iterable[object]: + streaming_config: Final[_StreamingRecognitionConfig] = riva_asr_module.StreamingRecognitionConfig( + config=recognition_config, interim_results=False + ) + return asr_service.streaming_response_generator( + audio_chunks=self._iter_audio_chunks(pcm_bytes), + streaming_config=streaming_config, + **self._timeout_kwargs(asr_service.streaming_response_generator, timeout, drop_params), + ) + + def _offline_responses( + self, + asr_service: _AsrService, + pcm_bytes: bytes, + recognition_config: _RecognitionConfig, + timeout: float | None, + ) -> Iterable[object]: + pending: Final = asr_service.offline_recognize(audio_bytes=pcm_bytes, config=recognition_config, future=True) + try: + return (pending.result(timeout=timeout),) + except Exception as e: + if pending.done(): + raise + pending.cancel() + raise NvidiaRivaException( + status_code=408, + message=f"NVIDIA Riva offline recognition did not finish within {timeout} seconds", + ) from e + + @classmethod + def _timeout_kwargs( + cls, rpc: Callable[..., object], timeout: float | str | httpx.Timeout | None, drop_params: bool + ) -> Mapping[str, float]: + # Forward the deadline so the call cannot block forever if the + # server stalls. Older riva-client versions do not accept a + # ``timeout`` kwarg, so pass it only when supported. + if timeout is None or not cls._supports_timeout_kwarg(rpc): + return MappingProxyType({}) + return MappingProxyType({"timeout": _deadline_seconds(timeout, drop_params)}) + @staticmethod def _supports_timeout_kwarg(callable_obj: Callable[..., object]) -> bool: try: @@ -429,9 +535,9 @@ class NvidiaRivaAudioTranscription: yield chunk @staticmethod - def _collect_final_results(stream) -> list[dict[str, object]]: + def _collect_final_results(stream: Iterable[object], require_final: bool = True) -> list[dict[str, object]]: """ - Walk the gRPC stream, ignore empty / non-final chunks, and return a + Walk the gRPC responses, ignore empty / non-final chunks, and return a list of normalized final-result dicts. Matching the user's note: the ``id`` blocks with no ``results`` are streaming heartbeats and must be skipped. @@ -440,7 +546,7 @@ class NvidiaRivaAudioTranscription: for response in stream: results = getattr(response, "results", None) or [] for result in results: - if not getattr(result, "is_final", False): + if require_final and not getattr(result, "is_final", False): continue alternatives = getattr(result, "alternatives", None) or [] if not alternatives: diff --git a/litellm/llms/nvidia_riva/common_utils.py b/litellm/llms/nvidia_riva/common_utils.py index b654e725dc8..c671b14e742 100644 --- a/litellm/llms/nvidia_riva/common_utils.py +++ b/litellm/llms/nvidia_riva/common_utils.py @@ -41,6 +41,9 @@ _GRPC_STATUS_CODE_TO_HTTP: Final[dict] = { } +_GRPC_MESSAGE_TOO_LARGE: Final = "message larger than max" + + def _extract_grpc_status_name(error: object) -> str | None: """ Best-effort extraction of a gRPC StatusCode name from an arbitrary error. @@ -79,8 +82,16 @@ def grpc_error_to_litellm_exception(error: Exception) -> NvidiaRivaException: a ``NvidiaRivaException`` with an appropriate HTTP-equivalent status code. """ status_name: Final = _extract_grpc_status_name(error) - http_status: Final = _GRPC_STATUS_CODE_TO_HTTP.get(status_name or "", 500) - detail: Final = _extract_grpc_details(error) or str(error) + if status_name == "RESOURCE_EXHAUSTED" and _GRPC_MESSAGE_TOO_LARGE in detail: + return NvidiaRivaException( + status_code=413, + message=( + f"NVIDIA Riva gRPC error ({status_name}): {detail}. The audio is too large for one offline " + "request to this Riva server: send shorter audio, or drop riva_offline on models that support streaming." + ), + ) + + http_status: Final = _GRPC_STATUS_CODE_TO_HTTP.get(status_name or "", 500) message = f"NVIDIA Riva gRPC error ({status_name}): {detail}" if status_name else f"NVIDIA Riva error: {detail}" return NvidiaRivaException(status_code=http_status, message=message) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 68cd8420166..7a186d84aa5 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -446,6 +446,7 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( # the request away from the admin's pinned configuration. "nvcf_function_id", "use_ssl", + "riva_offline", # Per-deployment opt-in that hands the whole call to the Rust core. It is a # deployment decision, not a request one: the Rust path uses its own client # rather than the one the deployment configured, and reports no post_call, diff --git a/tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py b/tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py index 7ecc0b47d9f..dbb5cde6060 100644 --- a/tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py +++ b/tests/unit/llms/nvidia_riva/audio_transcription/test_handler.py @@ -12,11 +12,12 @@ import io from types import SimpleNamespace from unittest.mock import MagicMock +import httpx import numpy as np import pytest import soundfile as sf - +import litellm from litellm.llms.nvidia_riva.audio_transcription import handler as handler_mod from litellm.llms.nvidia_riva.audio_transcription.handler import ( NvidiaRivaAudioTranscription, @@ -93,6 +94,24 @@ def mock_riva(monkeypatch): LINEAR_PCM = "LINEAR_PCM" streaming_responses_holder = {"value": []} + offline_response_holder = {"value": None} + offline_error_holder = {"value": None} + offline_future_holder = {"value": None} + streaming_calls = [] + offline_calls = {} + + class FakeRecognizeFuture: + def result(self, timeout: float | None = None) -> object: + offline_calls["timeout"] = timeout + if offline_error_holder["value"] is not None: + raise offline_error_holder["value"] + return offline_response_holder["value"] + + def done(self) -> bool: + return True + + def cancel(self) -> bool: + return False class FakeASRService: def __init__(self, auth): @@ -100,9 +119,15 @@ def mock_riva(monkeypatch): def streaming_response_generator(self, audio_chunks, streaming_config): # Drain audio_chunks generator so we exercise the chunking path. - list(audio_chunks) + streaming_calls.append(list(audio_chunks)) yield from streaming_responses_holder["value"] + def offline_recognize(self, audio_bytes: bytes, config: object, future: bool = False) -> object: + offline_calls["audio_bytes"] = audio_bytes + offline_calls["config"] = config + pending = offline_future_holder["value"] or FakeRecognizeFuture() + return pending if future else pending.result() + fake_riva_client = SimpleNamespace( Auth=FakeAuth, ASRService=FakeASRService, @@ -120,6 +145,11 @@ def mock_riva(monkeypatch): return SimpleNamespace( auth_calls=auth_calls, responses=streaming_responses_holder, + offline_response=offline_response_holder, + offline_error=offline_error_holder, + offline_future=offline_future_holder, + streaming_calls=streaming_calls, + offline_calls=offline_calls, client=fake_riva_client, ) @@ -203,6 +233,339 @@ def test_sync_path_aggregates_only_final_results(mock_riva, logging_obj): ) +def test_riva_offline_uses_unary_recognize_and_keeps_results_without_is_final( + mock_riva: SimpleNamespace, logging_obj: MagicMock +) -> None: + """ + Offline-only NIMs (Parakeet TDT, Whisper) reject StreamingRecognize. With + ``riva_offline`` the handler must call ``offline_recognize`` with the whole + resampled buffer and a bare ``RecognitionConfig``, and must keep every + result even though ``RecognizeResponse`` results carry no ``is_final``. + """ + mock_riva.offline_response["value"] = _fake_response( + results=[ + SimpleNamespace(alternatives=[_fake_alternative("Hello,", words=[_fake_word("Hello,", 0, 320)])]), + SimpleNamespace(alternatives=[_fake_alternative(" world.", words=[_fake_word("world.", 480, 870)])]), + ] + ) + + impl = NvidiaRivaAudioTranscription() + response: TranscriptionResponse = impl.audio_transcriptions( + model="nvidia/parakeet-0.6b-tdt", + audio_file=_make_wav_bytes(), + optional_params={ + "language_code": "multi", + "riva_offline": True, + "enable_word_time_offsets": True, + "response_format": "verbose_json", + "timestamp_granularities": ["word"], + }, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + + assert response.text == "Hello, world." + assert response["words"][1]["end"] == pytest.approx(0.87) + assert response._hidden_params["audio_transcription_duration"] == pytest.approx(1.0, abs=0.05) + assert mock_riva.streaming_calls == [] + assert len(mock_riva.offline_calls["audio_bytes"]) == pytest.approx(32000, abs=1600) + assert mock_riva.offline_calls["config"]._kwargs["language_code"] == "multi" + assert logging_obj.pre_call.call_args.kwargs["additional_args"]["complete_input_dict"]["riva_offline"] is True + + +def test_default_mode_streams_and_never_calls_offline_recognize( + mock_riva: SimpleNamespace, logging_obj: MagicMock +) -> None: + mock_riva.responses["value"] = [ + _fake_response(results=[_fake_result(is_final=True, alternatives=[_fake_alternative("streamed")])]) + ] + + impl = NvidiaRivaAudioTranscription() + response = impl.audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US"}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + assert response.text == "streamed" + assert mock_riva.offline_calls == {} + assert len(mock_riva.streaming_calls) == 1 + assert logging_obj.pre_call.call_args.kwargs["additional_args"]["complete_input_dict"]["riva_offline"] is False + + + +@pytest.mark.parametrize(("riva_offline", "expect_offline"), [("false", False), ("False", False), ("true", True)]) +def test_riva_offline_string_from_multipart_form_is_parsed( + mock_riva: SimpleNamespace, logging_obj: MagicMock, riva_offline: str, expect_offline: bool +) -> None: + mock_riva.responses["value"] = [ + _fake_response(results=[_fake_result(is_final=True, alternatives=[_fake_alternative("streamed")])]) + ] + mock_riva.offline_response["value"] = _fake_response( + results=[SimpleNamespace(alternatives=[_fake_alternative("offline")])] + ) + + response = NvidiaRivaAudioTranscription().audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US", "riva_offline": riva_offline}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + + assert response.text == ("offline" if expect_offline else "streamed") + assert (mock_riva.offline_calls != {}) is expect_offline + + +@pytest.mark.parametrize("riva_offline", ["yes", "", 1]) +def test_riva_offline_malformed_value_is_rejected_before_any_rpc( + mock_riva: SimpleNamespace, logging_obj: MagicMock, riva_offline: object +) -> None: + with pytest.raises(NvidiaRivaException) as excinfo: + NvidiaRivaAudioTranscription().audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US", "riva_offline": riva_offline}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + + assert excinfo.value.status_code == 400 + assert "riva_offline" in excinfo.value.message + assert mock_riva.offline_calls == {} + assert mock_riva.streaming_calls == [] + + +def test_riva_offline_malformed_value_is_dropped_under_drop_params( + mock_riva: SimpleNamespace, logging_obj: MagicMock +) -> None: + mock_riva.responses["value"] = [ + _fake_response(results=[_fake_result(is_final=True, alternatives=[_fake_alternative("streamed")])]) + ] + + response = NvidiaRivaAudioTranscription().audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US", "riva_offline": "yes"}, + litellm_params={"drop_params": True}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + + assert response.text == "streamed" + assert mock_riva.offline_calls == {} + + +def test_deployment_riva_offline_reaches_the_handler_through_transcription(mock_riva: SimpleNamespace) -> None: + mock_riva.offline_response["value"] = _fake_response( + results=[SimpleNamespace(alternatives=[_fake_alternative("offline")])] + ) + + response = litellm.transcription( + model="nvidia_riva/nvidia/parakeet-0.6b-tdt", + file=("clip.wav", _make_wav_bytes(), "audio/wav"), + api_base="localhost:50051", + riva_offline=True, + ) + + assert response.text == "offline" + assert mock_riva.streaming_calls == [] + assert mock_riva.offline_calls != {} + + +@pytest.mark.parametrize( + ("timeout", "waited_for"), + [("7.5", 7.5), (httpx.Timeout(30.0, read=7.5), 7.5), (httpx.Timeout(5.0, read=None), 600.0), (None, None)], +) +def test_offline_recognition_waits_the_timeout_in_seconds_whatever_its_form( + mock_riva: SimpleNamespace, timeout: str | httpx.Timeout | None, waited_for: float | None +) -> None: + mock_riva.offline_response["value"] = _fake_response( + results=[SimpleNamespace(alternatives=[_fake_alternative("offline")])] + ) + + response = litellm.transcription( + model="nvidia_riva/nvidia/parakeet-0.6b-tdt", + file=("clip.wav", _make_wav_bytes(), "audio/wav"), + api_base="localhost:50051", + riva_offline=True, + timeout=timeout, + ) + + assert response.text == "offline" + assert mock_riva.offline_calls["timeout"] == waited_for + + +@pytest.mark.parametrize("timeout", ["abc", "30s"]) +def test_malformed_timeout_on_offline_call_is_a_400_before_any_rpc( + mock_riva: SimpleNamespace, timeout: str +) -> None: + with pytest.raises(NvidiaRivaException) as excinfo: + litellm.transcription( + model="nvidia_riva/nvidia/parakeet-0.6b-tdt", + file=("clip.wav", _make_wav_bytes(), "audio/wav"), + api_base="localhost:50051", + riva_offline=True, + timeout=timeout, + ) + + assert excinfo.value.status_code == 400 + assert "timeout" in excinfo.value.message + assert "audio_bytes" not in mock_riva.offline_calls + + +def test_malformed_timeout_on_offline_call_waits_the_default_under_drop_params( + mock_riva: SimpleNamespace, +) -> None: + mock_riva.offline_response["value"] = _fake_response( + results=[SimpleNamespace(alternatives=[_fake_alternative("offline")])] + ) + + response = litellm.transcription( + model="nvidia_riva/nvidia/parakeet-0.6b-tdt", + file=("clip.wav", _make_wav_bytes(), "audio/wav"), + api_base="localhost:50051", + riva_offline=True, + timeout="abc", + drop_params=True, + ) + + assert response.text == "offline" + assert mock_riva.offline_calls["timeout"] == 600.0 + + +def test_malformed_timeout_still_streams_when_the_client_takes_no_timeout(mock_riva: SimpleNamespace) -> None: + mock_riva.responses["value"] = [ + _fake_response(results=[_fake_result(is_final=True, alternatives=[_fake_alternative("streamed")])]) + ] + + response = litellm.transcription( + model="nvidia_riva/nvidia/parakeet-ctc-1.1b", + file=("clip.wav", _make_wav_bytes(), "audio/wav"), + api_base="localhost:50051", + timeout="abc", + ) + + assert response.text == "streamed" + assert len(mock_riva.streaming_calls) == 1 + + +def test_oversized_offline_audio_is_a_non_retryable_413(mock_riva: SimpleNamespace, logging_obj: MagicMock) -> None: + class FakeGrpcError(Exception): + def code(self) -> SimpleNamespace: + return SimpleNamespace(name="RESOURCE_EXHAUSTED") + + def details(self) -> str: + return "SERVER: Received message larger than max (4348825 vs. 4194304)" + + mock_riva.offline_error["value"] = FakeGrpcError("rpc fail") + + with pytest.raises(NvidiaRivaException) as excinfo: + NvidiaRivaAudioTranscription().audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US", "riva_offline": True}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + + assert excinfo.value.status_code == 413 + assert litellm.utils.should_retry(excinfo.value.status_code) is False + + +def test_stalled_offline_recognition_is_cancelled_and_times_out_as_408( + mock_riva: SimpleNamespace, logging_obj: MagicMock +) -> None: + class FutureTimeoutError(Exception): + pass + + class StalledRecognizeFuture: + def __init__(self) -> None: + self.waited_for: float | None = None + self.cancelled = False + + def result(self, timeout: float | None = None) -> object: + self.waited_for = timeout + raise FutureTimeoutError() + + def done(self) -> bool: + return self.cancelled + + def cancel(self) -> bool: + self.cancelled = True + return True + + stalled = StalledRecognizeFuture() + mock_riva.offline_future["value"] = stalled + + with pytest.raises(NvidiaRivaException) as excinfo: + NvidiaRivaAudioTranscription().audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "en-US", "riva_offline": True}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=7.5, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + + assert excinfo.value.status_code == 408 + assert stalled.waited_for == 7.5 + assert stalled.cancelled is True + +def test_riva_offline_grpc_error_is_wrapped(mock_riva: SimpleNamespace, logging_obj: MagicMock) -> None: + class FakeGrpcError(Exception): + def code(self) -> SimpleNamespace: + return SimpleNamespace(name="INVALID_ARGUMENT") + + def details(self) -> str: + return "Unavailable model requested" + + mock_riva.offline_error["value"] = FakeGrpcError("rpc fail") + + impl = NvidiaRivaAudioTranscription() + with pytest.raises(NvidiaRivaException) as excinfo: + impl.audio_transcriptions( + model="m", + audio_file=_make_wav_bytes(), + optional_params={"language_code": "multi", "riva_offline": True}, + litellm_params={}, + model_response=TranscriptionResponse(), + timeout=60, + logging_obj=logging_obj, + api_key=None, + api_base="localhost:50051", + ) + assert excinfo.value.status_code == 400 + assert "INVALID_ARGUMENT" in excinfo.value.message + + def test_auth_nvcf_defaults_use_ssl_and_attaches_function_id(mock_riva, logging_obj): mock_riva.responses["value"] = [ _fake_response( @@ -343,9 +706,10 @@ def test_async_path_uses_to_thread(mock_riva, logging_obj): ) +@pytest.mark.parametrize("timeout", [12.5, "12.5", httpx.Timeout(60.0, read=12.5)]) def test_timeout_is_forwarded_to_streaming_generator_when_supported( - mock_riva, logging_obj -): + mock_riva: SimpleNamespace, logging_obj: MagicMock, timeout: float | str | httpx.Timeout +) -> None: """ Without a deadline the gRPC stream can block forever on a stalled Riva server. The handler must forward the call-level ``timeout`` to @@ -374,7 +738,7 @@ def test_timeout_is_forwarded_to_streaming_generator_when_supported( optional_params={"language_code": "en-US"}, litellm_params={}, model_response=TranscriptionResponse(), - timeout=12.5, + timeout=timeout, logging_obj=logging_obj, api_key=None, api_base="localhost:50051", diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 7597847ef6d..ee1ae43fd3a 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -3902,6 +3902,38 @@ class TestIsRequestBodySafeBlocksNVCFFunctionOverride: ) +def test_riva_offline_in_request_body_is_rejected() -> None: + with pytest.raises(ValueError, match="riva_offline"): + is_request_body_safe( + request_body={"model": "nvidia_riva/parakeet", "riva_offline": True}, + general_settings={}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + + +def test_admin_opt_in_proxy_wide_allows_riva_offline() -> None: + assert ( + is_request_body_safe( + request_body={"model": "nvidia_riva/parakeet", "riva_offline": True}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="nvidia_riva/parakeet", + ) + is True + ) + + +def test_api_base_override_keeps_the_deployments_riva_offline_mode() -> None: + from litellm.router_utils.clientside_credential_handler import get_dynamic_litellm_params + + out: Final = get_dynamic_litellm_params( + litellm_params={"model": "nvidia_riva/nvidia/parakeet-0.6b-tdt", "riva_offline": True}, + request_kwargs={"api_base": "caller-riva:50051"}, + ) + assert (out["api_base"], out["riva_offline"]) == ("caller-riva:50051", True) + + class TestIsRequestBodySafeBlocksRivaUseSsl: """``use_ssl`` is rejected as a request-body param unless the admin opted in proxy-wide or per-deployment.""" From dba4fa0e944e60299b88396282ccf3b73911c79f Mon Sep 17 00:00:00 2001 From: mrinal-berri Date: Tue, 20 Oct 2026 16:08:01 -0700 Subject: [PATCH 019/235] fix(proxy): flush tool discovery queue when use_redis_transaction_buffer is enabled (#47653) * fix(proxy): flush tool discovery queue when use_redis_transaction_buffer is enabled With the Redis transaction buffer on, discovered tools were never written to LiteLLM_ToolTable: the only call to _flush_tool_discovery_queue was on the non-Redis commit path. Each pod's tool queue is never buffered in Redis, so every pod now flushes its own queue on each spend tick, before the leader lock check. Co-authored-by: Noah Nistler <60981020+noahnistler@users.noreply.github.com> * test(proxy): exercise real leader election in the tool discovery flush test The writer's PodLockManager had no Redis in the test, so neither case was ever the leader and the lock-held parameter changed nothing. The fake Redis also had no async_rpush_pipeline, so every tick took the push-failure branch. Wire the fake into the lock manager the way ProxyLogging does in production, back it with real lists, and assert only the lock holder drains the spend buffer. Co-authored-by: Noah Nistler <60981020+noahnistler@users.noreply.github.com> --------- Co-authored-by: Noah Nistler <60981020+noahnistler@users.noreply.github.com> --- litellm/proxy/db/db_spend_update_writer.py | 2 + .../proxy/db/test_db_spend_update_writer.py | 56 ++++++++++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index e7909cbe600..bb6ffe675c0 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -1651,6 +1651,8 @@ class DBSpendUpdateWriter: window_spend_update_queue=self.window_spend_update_queue, ) + await self._flush_tool_discovery_queue(prisma_client=prisma_client) + # Only commit from redis to db if this pod is the leader if await self.pod_lock_manager.acquire_lock( cronjob_id=DB_SPEND_UPDATE_JOB_NAME, diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index d440b99a632..7437042e4bd 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -19,7 +19,7 @@ from redis.exceptions import DataError import litellm from litellm._logging import verbose_proxy_logger from litellm._service_logger import ServiceTypes -from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS +from litellm.constants import DB_SPEND_UPDATE_JOB_NAME, REDIS_UPDATE_BUFFER_KEY, SPEND_ROLLUP_LOCK_TIMEOUT_MS from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem from litellm.proxy.db.db_spend_update_writer import ( _TEAM_ADVISORY_LOCK_SQL, @@ -29,12 +29,14 @@ from litellm.proxy.db.db_spend_update_writer import ( _SpendTableName, ) from litellm.proxy.db.db_transaction_queue.daily_spend_update_queue import DailySpendUpdateQueue +from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager from litellm.proxy.db.db_transaction_queue.redis_update_buffer import RedisUpdateBuffer from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdateQueue from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, ) from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL +from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -2365,6 +2367,58 @@ async def test_commit_spend_updates_uses_pipeline(): mock_redis_update_buffer.get_all_daily_tag_spend_update_transactions_from_redis_buffer.assert_not_called() +class _InMemoryRedis: + def __init__(self) -> None: + self.store: dict[str, str] = {} + self.lists: dict[str, list[object]] = {} + + async def async_set_cache(self, key: str, value: str, nx: bool = False, **kwargs: object) -> bool | None: + if nx and key in self.store: + return None + self.store[key] = json.dumps(value) + return True + + async def async_get_cache(self, key: str, **kwargs: object) -> str | None: + raw = self.store.get(key) + return json.loads(raw) if raw is not None else None + + async def async_delete_cache(self, key: str) -> int: + return 1 if self.store.pop(key, None) is not None else 0 + + async def async_rpush_pipeline(self, rpush_list: list[RedisPipelineRpushOperation]) -> list[int]: + for op in rpush_list: + self.lists.setdefault(op["key"], []).extend(op["values"]) + return [len(self.lists[op["key"]]) for op in rpush_list] + + async def async_lpop_pipeline(self, lpop_list: list[RedisPipelineLpopOperation]) -> list[list[object] | None]: + return [self.lists.pop(op["key"], None) for op in lpop_list] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("lock_held_by_another_pod", [False, True]) +async def test_commit_with_redis_writes_discovered_tools_on_leader_and_non_leader(lock_held_by_another_pod: bool): + redis = _InMemoryRedis() + if lock_held_by_another_pod: + assert await PodLockManager(redis_cache=redis).acquire_lock(cronjob_id=DB_SPEND_UPDATE_JOB_NAME) + db_writer = DBSpendUpdateWriter(redis_cache=redis) + db_writer.pod_lock_manager.redis_cache = redis + db_writer.tool_discovery_queue.add_update({"tool_name": "get_weather", "origin": "user_defined"}) + prisma_client = MagicMock() + prisma_client.db.litellm_tooltable.upsert = AsyncMock() + + await db_writer._commit_spend_updates_to_db_with_redis( + prisma_client=prisma_client, + n_retry_times=0, + proxy_logging_obj=MagicMock(), + ) + + upserted = [c.kwargs["where"] for c in prisma_client.db.litellm_tooltable.upsert.await_args_list] + assert upserted == [{"tool_name": "get_weather"}] + assert (REDIS_UPDATE_BUFFER_KEY in redis.lists) is lock_held_by_another_pod, ( + f"spend buffer left in redis={list(redis.lists)}, only the lock holder may drain it" + ) + + @pytest.mark.asyncio async def test_commit_with_redis_requeues_all_on_db_failure(): """ From 7c39dbfab8cefd91e96416ea32d800a44785f813 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 20 Oct 2026 16:12:28 -0700 Subject: [PATCH 020/235] build(deps): bump pillow and dashboard vite, defer certifi to clear osv-scan (#47678) Pillow 13.0.0 requires Python >= 3.11, so the ci group pin is split by Python version and the 3.10 resolution stays on 12.3.0 with a 7-day osv-scanner override. Dashboard vite moves from 7.3.5 to 7.3.7. The certifi fix (2026.10.19) is still inside the 3-day uv exclude-newer window, so its advisory is ignored until 2026-10-23 --- osv-scanner.toml | 13 ++++ pyproject.toml | 3 +- ui/litellm-dashboard/package-lock.json | 10 +-- ui/litellm-dashboard/package.json | 2 +- uv.lock | 90 ++++++++++++++++++++++++-- 5 files changed, 107 insertions(+), 11 deletions(-) diff --git a/osv-scanner.toml b/osv-scanner.toml index 60f163392a8..bb802137f6e 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -7,3 +7,16 @@ reason = "diskcache has no fixed release published; remove this entry once one e id = "GHSA-vfj7-8cjw-p6xm" ignoreUntil = 2026-11-03 reason = "braces has no fixed release published (3.0.3, 2024-05-21, is latest and last_affected); dev-only via knip > fast-glob > micromatch; remove this entry once one exists" + +[[IgnoredVulns]] +id = "GHSA-6q48-6c7w-9gw2" +ignoreUntil = 2026-10-23 +reason = "certifi 2026.10.19 (published 2026-10-20) is inside the 3-day uv exclude-newer window until 2026-10-23 02:24 UTC; bump certifi and remove this entry once it clears" + +[[PackageOverrides]] +name = "pillow" +version = "12.3.0" +ecosystem = "PyPI" +vulnerability.ignore = true +effectiveUntil = 2026-10-27 +reason = "pillow 13.0.0 requires Python >= 3.11, so the Python 3.10 resolution of the ci and dev groups stays on 12.3.0; 3.11+ is on 13.0.0" diff --git a/pyproject.toml b/pyproject.toml index a11708b15f2..9076760153a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -269,7 +269,8 @@ ci = [ # protobuf, Pillow is a compiled C extension). "tenacity==8.5.0", "google-generativeai==0.8.6", - "Pillow==12.3.0", + "Pillow==12.3.0; python_version == '3.10'", + "Pillow==13.0.0; python_version >= '3.11'", # Azure batch E2E tests still import psycopg2 directly. "psycopg2-binary==2.9.11", "pytest-codspeed==4.3.0", diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 7cc7415d52d..944455721cd 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -86,7 +86,7 @@ "tw-animate-css": "1.4.0", "typescript": "5.9.3", "typescript-eslint": "8.60.1", - "vite": "7.3.5", + "vite": "7.3.7", "vitest": "4.1.11" }, "engines": { @@ -12422,13 +12422,13 @@ } }, "node_modules/vite": { - "version": "7.3.5", - "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.5.tgz", - "integrity": "sha512-KuOaNhcnGFN2zIPGA7wRmzF+lJA1sea7rHq17aiJ++9lzY1WWG6Jpwqwe1KNbRVPIqHmr8GLYx7jbrQcN/7/ww==", + "version": "7.3.7", + "resolved": "https://registry.npmjs.org/vite/-/vite-7.3.7.tgz", + "integrity": "sha512-AHEzTORMbi7/7aXp7w7KZ5+DCUdSeLsJz0grY8xHpc9DHdYb0vQu6DqJfm2tOw3gDExNgPgnJTXxM6syYFHaCA==", "dev": true, "license": "MIT", "dependencies": { - "esbuild": "^0.27.0", + "esbuild": "^0.27.0 || ^0.28.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index cd08995471f..f2a1a5e4e7f 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -103,7 +103,7 @@ "tw-animate-css": "1.4.0", "typescript": "5.9.3", "typescript-eslint": "8.60.1", - "vite": "7.3.5", + "vite": "7.3.7", "vitest": "4.1.11" }, "overrides": { diff --git a/uv.lock b/uv.lock index afc377eb7e4..03403828225 100644 --- a/uv.lock +++ b/uv.lock @@ -4779,7 +4779,8 @@ ci = [ { name = "logfire" }, { name = "lunary", version = "1.4.36", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, { name = "lunary", version = "1.4.37", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, - { name = "pillow" }, + { name = "pillow", version = "12.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pillow", version = "13.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "psutil" }, { name = "psycopg2-binary" }, { name = "pyarrow" }, @@ -5001,7 +5002,8 @@ ci = [ { name = "logfire", specifier = "==4.6.0" }, { name = "lunary", marker = "python_full_version == '3.10.*'", specifier = "==1.4.36" }, { name = "lunary", marker = "python_full_version >= '3.11'", specifier = "==1.4.37" }, - { name = "pillow", specifier = "==12.3.0" }, + { name = "pillow", marker = "python_full_version == '3.10.*'", specifier = "==12.3.0" }, + { name = "pillow", marker = "python_full_version >= '3.11'", specifier = "==13.0.0" }, { name = "psutil", specifier = "==7.2.2" }, { name = "psycopg2-binary", specifier = "==2.9.11" }, { name = "pyarrow", specifier = "==23.0.1" }, @@ -5498,7 +5500,8 @@ dependencies = [ { name = "numpy", version = "1.26.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, { name = "numpy", version = "2.4.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, { name = "packaging" }, - { name = "pillow" }, + { name = "pillow", version = "12.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pillow", version = "13.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, { name = "pyparsing" }, { name = "python-dateutil" }, ] @@ -7174,6 +7177,9 @@ wheels = [ name = "pillow" version = "12.3.0" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.11'", +] sdist = { url = "https://files.pythonhosted.org/packages/1c/3d/bb7fca845737cf9d7dbde16ed1843984665ff2e0a518f5db43e77ec540b9/pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce", size = 47025035, upload-time = "2026-07-01T11:56:38.965Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/25/c2/669d88644cddb1485bd9534e63e8cf476c8e51cb3c3a1297677023505c0e/pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a", size = 5392418, upload-time = "2026-07-01T11:53:27.808Z" }, @@ -7243,6 +7249,81 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/54/0169bc772ec491108b62f644f8ecf1fe5d8ae5ebafde2ee2142210166903/pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a", size = 7231786, upload-time = "2026-07-01T11:56:35.046Z" }, ] +[[package]] +name = "pillow" +version = "13.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.14'", + "python_full_version == '3.13.*'", + "python_full_version == '3.12.*'", + "python_full_version == '3.11.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/ce/a4/9cc30690a394a8305e8be4c9036aa16d0dfddadd09079860b582f562496d/pillow-13.0.0.tar.gz", hash = "sha256:cdcd1ba258178f7356d74170c9a81ba4eab6a5d55748bc402a5a2a6fe6be3510", size = 47058631, upload-time = "2026-10-16T22:18:09.718Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/d7/af6af1b8cc55e07cb0489d5b0950bf5a9bb1461eeba825700b6d8d0e26ea/pillow-13.0.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:6f38f3f885a09cafc93fd667b1173a92f379e43785d635e7983c893f0da319fb", size = 5592324, upload-time = "2026-10-16T22:15:24.672Z" }, + { url = "https://files.pythonhosted.org/packages/0b/d2/2c12cb4fbb5d95f38d1001ff3d95fafcb830077018cba40e56a8f932dfbc/pillow-13.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37290fbef0c73e2d51d594eb78fded7dc7170d1c3ad0a890269e6fd8c8c4e061", size = 4856298, upload-time = "2026-10-16T22:15:27.669Z" }, + { url = "https://files.pythonhosted.org/packages/5c/ee/229b27472483a4801ea94990815e710391836afdf987b4b722b62bd75db6/pillow-13.0.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c88c10dd432f3b39ccdafc9ed81d0c21b8fd634502574da76d1bfa4fe2c76d0", size = 6517304, upload-time = "2026-10-16T22:15:29.43Z" }, + { url = "https://files.pythonhosted.org/packages/36/46/f509b74787f40ce0627501089d630b2c741b63b244c1f40b640b3b01e181/pillow-13.0.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a625df9a2a5597fa5e5266e01aefc82604bb85d291e3547981e863560d75da6c", size = 7140667, upload-time = "2026-10-16T22:15:31.368Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/5a398b8ec2c37145397d6461c0b551c23ec03072aa64f357aed25e582b00/pillow-13.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:10b5019883c98f1dd819ca48ffc6f33a91cee04948fe637c605f4345c7cafd6f", size = 6610583, upload-time = "2026-10-16T22:15:34.025Z" }, + { url = "https://files.pythonhosted.org/packages/4f/7b/f2f1109b230fd4e2d57ab2305de31adc37ea9b4ff8719436c623cb484415/pillow-13.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f391dc92a015c478745192fe4f2e120c2af2aba18d9fe0616341cbf537599806", size = 7218241, upload-time = "2026-10-16T22:15:36.214Z" }, + { url = "https://files.pythonhosted.org/packages/ab/d3/069edb9083b267746baf59756282cacef904d3475c5288c7216fb5195a98/pillow-13.0.0-cp311-cp311-win32.whl", hash = "sha256:6a0920cb55df5a4d3f6862964908078c0bd59332bdd1bcd20d14e8c70ef27cc5", size = 6515278, upload-time = "2026-10-16T22:15:38.129Z" }, + { url = "https://files.pythonhosted.org/packages/7d/f6/6e2d96413e4c9dbec12ea584d9e646d602877d4cce3ea669099f959deac6/pillow-13.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:d7710eb676e9bb68965cb045be0d333b4c0d7f9dfbaa7fbf121385b1588683d3", size = 7278471, upload-time = "2026-10-16T22:15:40.286Z" }, + { url = "https://files.pythonhosted.org/packages/70/e7/baff21f8cb60496c8c250609501b88919eeedb4088d3da106f6c62f3f3c0/pillow-13.0.0-cp311-cp311-win_arm64.whl", hash = "sha256:12fbb851cf323cc19aac3eea1df5ad17b57e91e9fe3cfd06100dba7bb8208c13", size = 2698981, upload-time = "2026-10-16T22:15:42.637Z" }, + { url = "https://files.pythonhosted.org/packages/7a/b8/58d79332229482eb0d27c82973850808c19d37e5bdc14443d3417bd373f5/pillow-13.0.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:67c8d44ad89fa7eb88de5d6ef6199b1e03dd24974f85a67639fc8badb801fe79", size = 5568643, upload-time = "2026-10-16T22:15:44.474Z" }, + { url = "https://files.pythonhosted.org/packages/39/4b/f2ed53d83aa44ac19599ac0a54897f867fafb95f406980d1e24083f8cbe3/pillow-13.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc481f742ccda20e9ed760b8f978dff50f2a4e9bc253ac167fb0a05b0783ef52", size = 4852329, upload-time = "2026-10-16T22:15:46.284Z" }, + { url = "https://files.pythonhosted.org/packages/3b/bb/9e30e8ed18c20800ee327a12107d5249ad8dc444867b3adfcc21654e3c71/pillow-13.0.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8bf295fe459343bc6093c8e5e0df60bfa189e0fcb775a22e2c2488fdeccc6c0d", size = 6519309, upload-time = "2026-10-16T22:15:48.14Z" }, + { url = "https://files.pythonhosted.org/packages/e6/99/fdf65cc539e266ab93546c261bdf98411fe165c6e4e13f9dbf597097192b/pillow-13.0.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:42d63be3af4f9b951b199b0a9d23365d37ab14687e6d77b74940432c8515c4ca", size = 7144542, upload-time = "2026-10-16T22:15:50.093Z" }, + { url = "https://files.pythonhosted.org/packages/26/9b/f38b7c4d8f599537e2d1a6d546cec9beef7536ff8f5c49a3f9c78179d8ac/pillow-13.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:4bd024d567eca931b668978d672dba914d4b8e2628e2c3073a2ad178de8a2620", size = 6618042, upload-time = "2026-10-16T22:15:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7c/45ee28531f89ee4582967d8a17f7a26d71ca64cd902b93c269590cd60a00/pillow-13.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23edf4ec4b6f4f240e8df6e4985e9ee2ced86474632433e55fb1e2548744bf40", size = 7228137, upload-time = "2026-10-16T22:15:54.009Z" }, + { url = "https://files.pythonhosted.org/packages/01/bc/9c2a969cb1b70201d04fa17fd5aedf02a732870b8da86bceaaa30b0ca385/pillow-13.0.0-cp312-cp312-win32.whl", hash = "sha256:02464da86f561f410e13b061ec37c3068a9cb557a095494157742d07b3c24a27", size = 6515423, upload-time = "2026-10-16T22:15:56.25Z" }, + { url = "https://files.pythonhosted.org/packages/ff/cd/73a79ecdc822076002eb24b5e1e4a530b0bb5310fcb6d239abbec877110b/pillow-13.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:ff3fe9eb621fbb9db4c4be33ab946f6bac5f98ab50ca8d3260503653656c961b", size = 7272166, upload-time = "2026-10-16T22:15:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/4a/a3/be5a1236efd085b3984e4305da8f179ae9dab80c991d2ad78f2bd5c4c843/pillow-13.0.0-cp312-cp312-win_arm64.whl", hash = "sha256:c3bc8ee0271387e0b1e3ad0ff893645990a656e79bf9c42f4f734a3642b11d51", size = 2698943, upload-time = "2026-10-16T22:16:00.637Z" }, + { url = "https://files.pythonhosted.org/packages/82/e2/9132d38df320e8099e2642d9fa538f18a7dfebf68a875a31725dc2a115aa/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e98b4f91f99b24b1da72e10621ba181f7281922384a7403094debb4199de6c00", size = 4190620, upload-time = "2026-10-16T22:16:02.527Z" }, + { url = "https://files.pythonhosted.org/packages/cb/be/436dac0996876f65ef1916ffb711c5ef4af2ebc57414b6f16642e07f6b47/pillow-13.0.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:34fb3db1149c879343861831728c243c432d4a5afc8f1878ca7612ef8de2a033", size = 4290286, upload-time = "2026-10-16T22:16:04.861Z" }, + { url = "https://files.pythonhosted.org/packages/60/16/7c1007162e4169ea0d5bec47e2e0b2174f56c50cf5b8c02a1dc23e41037e/pillow-13.0.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:2d50500a4a3bc6469adcdb319cf05d688b08ca3bf0f22a05339f3b25b433a599", size = 3723169, upload-time = "2026-10-16T22:16:06.956Z" }, + { url = "https://files.pythonhosted.org/packages/08/bc/578ab2eaa04705a8978bd9be0fe15c3d43a8fc95097c4f7e39540e79e4fa/pillow-13.0.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ef5fe6b7db3f501c56000ba8fe7c7e1a35ca1d45824fbeb95a67daa85cff4dbe", size = 5568469, upload-time = "2026-10-16T22:16:08.664Z" }, + { url = "https://files.pythonhosted.org/packages/98/c8/ccede3dcc4f274b80fb49199eed1ef34ba05566c8b71976b9d1f111da2e3/pillow-13.0.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f11a3c1358bb3c68012a7b1f41b223c3db9f7bfde90b481df028cfe1c3363313", size = 4852187, upload-time = "2026-10-16T22:16:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/fb/3c/cd0770e96d20bfd990db024bcbd43897ff3e88da1f4782cf90ea231b1d4f/pillow-13.0.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19b66ca0a4d25cc0200b381ce824c7ea090db61427315e73cfb05dbf1b35c369", size = 6515908, upload-time = "2026-10-16T22:16:12.896Z" }, + { url = "https://files.pythonhosted.org/packages/ef/fe/f5cdaa2d1c4086e757430f368c271a58ee29cfe7378e7ae7b9d964015c25/pillow-13.0.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5f56499c1a74d2b862beb0c33b8016f99d095858455df9d21583bbbc1c15b582", size = 7142544, upload-time = "2026-10-16T22:16:14.844Z" }, + { url = "https://files.pythonhosted.org/packages/fb/b5/a54f6d44adbcd23737c8c12953f42be107e4cca64e247c0ca3a5b563061c/pillow-13.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2a14cd73d5e6c927f644b0110ee81f59c4ef506259ec9183904747f2caacc81", size = 6613097, upload-time = "2026-10-16T22:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/e6/b4/9e6c1207c21b2e91db0d15c16d30f3701ce5add1bd4da0e787761e66cb6e/pillow-13.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08b826a9bf1265c76de04a42061d61ea6b109418b7826f28ff1ecdfaa26c5502", size = 7224846, upload-time = "2026-10-16T22:16:19.195Z" }, + { url = "https://files.pythonhosted.org/packages/78/9f/d3c3cb0518f3dd1eee0ef501535f53d15597a53c5c4118d178086143397a/pillow-13.0.0-cp313-cp313-win32.whl", hash = "sha256:12b0fa15879fe5a9d9eebe66e4e2d74128a773b1a4448010c9c714f45e04b9d8", size = 6516013, upload-time = "2026-10-16T22:16:21.389Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ed/1b16fb66a37e1b9e41370b93b2c614c7da761a5b2a41dc333f7af4639962/pillow-13.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:9dda0788a9932776a75d8bcc020076edc308ce2508b51f6739a45e3158b3fc42", size = 7284777, upload-time = "2026-10-16T22:16:23.393Z" }, + { url = "https://files.pythonhosted.org/packages/87/7d/8d02eba9f7ecc87f8e90f731e032cc4773802f2a843763afd0c66a535ff4/pillow-13.0.0-cp313-cp313-win_arm64.whl", hash = "sha256:40a266371ece581e5de13b6635ef78d7487824af80561b6ca27694a88c19bc0f", size = 2698826, upload-time = "2026-10-16T22:16:25.382Z" }, + { url = "https://files.pythonhosted.org/packages/8e/45/fbad9c7f5e2f1745c373a434d46072bc84eddfefaa28b852425ac957fda6/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:44629d4498c9cb6b2051806155f22141df0d8a929c56134b166655232dfb2f8c", size = 4190721, upload-time = "2026-10-16T22:16:27.524Z" }, + { url = "https://files.pythonhosted.org/packages/a2/bf/2074902a259a90692c5dbf132885d21671644cbbd3f690047516243a55db/pillow-13.0.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:bf97ba1b5542050e9b0c74e632323b6d8136cc99d64e193abebee22fe6013c76", size = 4290236, upload-time = "2026-10-16T22:16:29.562Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a5/3d1bb93b8ea38d15d21356c1b8e9f50971ac592aaa248f7fb71739094c0e/pillow-13.0.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:aa694c49787040943570d61c9b8fc01271bd2a2cfdd55f0d896f8aecdce1a981", size = 3723093, upload-time = "2026-10-16T22:16:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/84/a1/be060135d85187b25fa8b2573786c0e45c62ae27527fff5257675eb97f4a/pillow-13.0.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:49c12af515073e9bf511e4ef736743bf70d531a643668722279314b937742ba4", size = 5572253, upload-time = "2026-10-16T22:16:33.103Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f4/0bf6f1ac9ba7ee2ffe9914aeb410a1035a0c1a3c42f122388d7fc950bc4b/pillow-13.0.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d9b0a2bee3f3e03e9f29057aedb807dbf1e1d96ee1b9d0f422d9db6c8583271d", size = 4852269, upload-time = "2026-10-16T22:16:35.102Z" }, + { url = "https://files.pythonhosted.org/packages/83/16/b603274617b3e6728da3a0133d4fba1ea8d7fbe0c4ea5fef9cbffa398b8b/pillow-13.0.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5933fcea5a5ba64c686f55c886a115cbfd92f2900460784a1e79909fdbafd7b6", size = 6516058, upload-time = "2026-10-16T22:16:37.228Z" }, + { url = "https://files.pythonhosted.org/packages/1d/f2/1371707ce438ba9a43f9d24089828494ec6f9396e28cd7903aa704dd6306/pillow-13.0.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e91c2b1cbeae768c98752e67e717e283d68dae45717b41f2d91eaee648e37b7d", size = 7142574, upload-time = "2026-10-16T22:16:39.172Z" }, + { url = "https://files.pythonhosted.org/packages/b9/a9/62cf68587f910bc87c5951bcb6e9430c7643a65076adf5f18b4bf350e2bd/pillow-13.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7f6299e494e46253d40c34653cac369412f44ca53b5f85d848d3f95b1ed0546c", size = 6614295, upload-time = "2026-10-16T22:16:41.117Z" }, + { url = "https://files.pythonhosted.org/packages/6c/c9/9e9b48178649730299be0bafbf5f810a5b6977538f5a92d47e4ba5f5d608/pillow-13.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f9bd7d260d986f3553518c5d723544346fa48031f8aef3265d786aeb59845bc0", size = 7224841, upload-time = "2026-10-16T22:16:43.118Z" }, + { url = "https://files.pythonhosted.org/packages/8f/48/c44e91384b1b6c072a714a63d00608acc16a6a968735e478e4be4384e5ac/pillow-13.0.0-cp314-cp314-win32.whl", hash = "sha256:2e31c4e68cffd7b1f116127626813b82fcba19edf6b940a7b0c266d6484d0055", size = 6508694, upload-time = "2026-10-16T22:16:45.883Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/7ad17747517e4c813dca78257d6bb593466f8231baf1384bc98d92745edb/pillow-13.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:f7cbb7a64e7ec664775195e4f81923bc8a93d9451b963f844e59118f63147a7a", size = 7281830, upload-time = "2026-10-16T22:16:47.725Z" }, + { url = "https://files.pythonhosted.org/packages/41/9f/e5b51f2ce8ef4563215334035403ce61a244f93a2e65c24a940131618f87/pillow-13.0.0-cp314-cp314-win_arm64.whl", hash = "sha256:a6aaa7871aebca081eb5c45f83e845bbd9712ae99fa9286dbfdcee835e63f4b4", size = 2695468, upload-time = "2026-10-16T22:16:49.515Z" }, + { url = "https://files.pythonhosted.org/packages/97/73/a53502f731f6c84805a4ce87a8881e86d07a65aee64728b900d112dc703e/pillow-13.0.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7b999816b5850a35ada08443e6e03945f1f6488144a83afe2d37b240a552aafa", size = 5578196, upload-time = "2026-10-16T22:16:51.291Z" }, + { url = "https://files.pythonhosted.org/packages/21/50/7e949959a56b5559c78ef172b70fc6025706acb289f3f648a15329e4fa6f/pillow-13.0.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:67871a13cf47275201553b536c025da777f997fa562a19eb798af4ad4b97c18f", size = 4857276, upload-time = "2026-10-16T22:16:53.399Z" }, + { url = "https://files.pythonhosted.org/packages/c9/52/43ded0ac08e77e39d17bd6d894162572201b5e8fdc4a022b14c55a1960a4/pillow-13.0.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e8ca8f69b7c77fa96dad99f8638c1fc4f1c0bed4acaa1baf7a3c31bdb80a0ab1", size = 6607008, upload-time = "2026-10-16T22:16:55.525Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2e/4a57ed216d5bf877387cba4d44c02aa87abe51ad2e06404aa6b597c4da39/pillow-13.0.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6bce6a29c3898191f4c9998e4d4e8f69790caeabde46e54da4bc235ce5bde683", size = 7224281, upload-time = "2026-10-16T22:16:57.738Z" }, + { url = "https://files.pythonhosted.org/packages/5c/3d/2929b248e135752d63aaeabf53f6de797e905b7b6263f9c1e2adade20aba/pillow-13.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:816752a5ef76a80484c8c441c85593b7ea9304f6231fc5776ad1fbff7e773058", size = 6695600, upload-time = "2026-10-16T22:16:59.842Z" }, + { url = "https://files.pythonhosted.org/packages/bc/33/fb0da992cf084ed1ee6ee05e96b6191e35c351eb707ca2df874feb24ce47/pillow-13.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5bdb38ba220b61b4ec31dd1046dea6b016d3c6850a23f43ed8b58332b2fc668e", size = 7300589, upload-time = "2026-10-16T22:17:01.686Z" }, + { url = "https://files.pythonhosted.org/packages/bb/af/c30dd9ef6a3dce84ecb1b2393cac629047a32e66279056c6cb46c3ffd65c/pillow-13.0.0-cp314-cp314t-win32.whl", hash = "sha256:06903aac074d00c988e77a1fc435286dc498506d4ca8d6f69df9babddf4ecc15", size = 6517204, upload-time = "2026-10-16T22:17:03.722Z" }, + { url = "https://files.pythonhosted.org/packages/85/d5/ef3a8ae7609c8ab26570abf18194882534b480d245a9148b895a2262c004/pillow-13.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:9a635068d9793c7f0d929f39f18b58f1bd86c236e8e7b64167b86af9232ebca0", size = 7286695, upload-time = "2026-10-16T22:17:05.543Z" }, + { url = "https://files.pythonhosted.org/packages/84/3c/0861db0c69a03689ebe3e7599d35668a414deb5aaafd8255215e4e62e3d3/pillow-13.0.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0d1a9cf7bc6917538b20f8d06b9ab2147e542f68849e96fc58eb66bdf6acc692", size = 2698098, upload-time = "2026-10-16T22:17:07.674Z" }, + { url = "https://files.pythonhosted.org/packages/68/c0/fb7b1caf25ed5ac5d46116f09b8b0706b9559833c65c667cecb1a53153f2/pillow-13.0.0-pp311-pypy311_pp80-macosx_10_15_x86_64.whl", hash = "sha256:405be6058e206a86fedce2f715ecc2343f94072d7925251632dffa2c05f70dca", size = 5521630, upload-time = "2026-10-16T22:17:50.008Z" }, + { url = "https://files.pythonhosted.org/packages/1a/e6/4d11d5b1137f4fc080605cce410cfaae4f303ad400bdec75d27d3083fa29/pillow-13.0.0-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:b84be41387fc00db28c13c02e30d42b02d8d8f6a4a741a1aee6012ca7f1f6039", size = 4806023, upload-time = "2026-10-16T22:17:51.932Z" }, + { url = "https://files.pythonhosted.org/packages/f1/45/0923202a658e8ff6aafd72c1a4395ebadcdebaa182c437d8595d3dd3db32/pillow-13.0.0-pp311-pypy311_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a192360d512305fe92d3053e4f7627bbad1f7708f1f4ad38cccd9336acabe9ad", size = 5464831, upload-time = "2026-10-16T22:17:53.742Z" }, + { url = "https://files.pythonhosted.org/packages/07/52/d3e38f0cad1dadea2a658b9a7d6ba8a04048f5ae14f011b806e5c3bd6bcb/pillow-13.0.0-pp311-pypy311_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:83b37abd87b590acd8f47ca8a61f70893322b11834d36e07c2298dacad4d14d2", size = 6043133, upload-time = "2026-10-16T22:17:55.625Z" }, + { url = "https://files.pythonhosted.org/packages/c4/f9/2ed7685980c1d64f8131a7d87a19c369424747577152622b73ddeff163b7/pillow-13.0.0-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:73ee25cf8cf8b75d5a5aeed7cb6588f5e5dbfbb0fde51707b630bbb23897645c", size = 7365208, upload-time = "2026-10-16T22:17:57.509Z" }, + { url = "https://files.pythonhosted.org/packages/60/da/f6d644e631c5b370c5d0b899f3ccadd2780aab070b29a110527cf4e1fd2c/pillow-13.0.0-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:8619685cad8efc4c38c66d3b36aad854523297c43d748b3b6f0cc03c90ba3463", size = 5523188, upload-time = "2026-10-16T22:17:59.334Z" }, + { url = "https://files.pythonhosted.org/packages/e4/b0/2594f202d743a35d4fe80682bfd96f277e5ca7aecb27cf64ef5bfc1d000b/pillow-13.0.0-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:7e91e97a929232cdb828889ee6ad0e48d7f40f4ec38818f4e4a8968b2b1af22e", size = 4806808, upload-time = "2026-10-16T22:18:01.188Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ed/d504165ef79af6fafe25c76726cf30f544cc236002f7ed054f1d51adc4ff/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:93ff9c9c3663ce23aaa5d0b6b6abac55359e2a0c06c7e30f3771829af816be63", size = 5464358, upload-time = "2026-10-16T22:18:02.779Z" }, + { url = "https://files.pythonhosted.org/packages/7b/fe/a10aba5503ca13e9e009b8bd0a1a4ce5584bdce246245fd6f3a83701369c/pillow-13.0.0-pp312-pypy312_pp80-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e65dd6e75de67b2362a4fa3e16aa627595f65054dc4ec73d64680c84d592fa61", size = 6043514, upload-time = "2026-10-16T22:18:04.538Z" }, + { url = "https://files.pythonhosted.org/packages/c0/37/66b12dc476041fe523bca29c9d58fbf1ea92a4eaf73acfb9e993948f1191/pillow-13.0.0-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:567f20f1c75897d7663830da6ea8af719e03d712b1d9ee7f813319f9ece42e5a", size = 7366030, upload-time = "2026-10-16T22:18:06.638Z" }, +] + [[package]] name = "platformdirs" version = "4.9.6" @@ -8698,7 +8779,8 @@ version = "5.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "charset-normalizer" }, - { name = "pillow" }, + { name = "pillow", version = "12.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, + { name = "pillow", version = "13.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/4a/51/dbe28534ae12c852f61be91f039f343305fd1f34f1c66b8de75afae7a525/reportlab-5.0.1.tar.gz", hash = "sha256:ebd13154be1c8515e665de70bd2d303ae9ddc3ef47e44afd5116441ca0283a26", size = 3945711, upload-time = "2026-08-20T13:48:16.461Z" } wheels = [ From 480ee653c518bd4bed8a43c7231115dc99ce3f4a Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:22:21 +0000 Subject: [PATCH 021/235] fix(responses): send the billed background response cost to the creating key's and team's loggers (#47506) * test(responses): the billed background response cost reaches the creating key's and team's loggers * fix(responses): send the billed background response cost to the creating key's and team's loggers * fix(responses): bill an unreadable or rejected creator through proxy loggers, redact and honor disabled callbacks on the billed event * fix(responses): follow the creator's message logging setting on the billed event and read team logging for non-key creators The billed event no longer forces redaction. It follows the creating key's or team's turn_off_message_logging, else the proxy's, and redacts when the creator cannot be read or its settings are rejected. A creator without a virtual key hash bills through its team's settings. function_setup keeps its original callback split, and the billed path uses a separate non-mutating helper. * fix(responses): bill rotated, deleted, and custom auth creators through their own logging and retry failed creator lookups * fix(responses): follow a key's whole grace-period rotation chain to the live key for its billed charge's logging * fix(responses): bill a background response whose creator cannot be read, without the creator's settings * docs(enterprise): say the background response row is claimed before its creator is read * fix(responses): keep the loaded creator rows' budgets and aliases when one creator lookup fails --- .../proxy/common_utils/check_batch_cost.py | 10 +- .../common_utils/check_responses_cost.py | 91 ++- .../managed_object_attribution.py | 185 +++++- litellm/proxy/litellm_pre_call_utils.py | 48 +- litellm/utils.py | 16 + .../test_background_response_billed_once.py | 83 ++- .../test_responses_background_cost.py | 6 + .../common_utils/test_check_responses_cost.py | 581 +++++++++++++++++- .../response_api_endpoints/test_endpoints.py | 3 + 9 files changed, 936 insertions(+), 87 deletions(-) diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py index bd9440d4754..df1f263a947 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_batch_cost.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Final, List, Literal, Optional, Protocol, Tupl from litellm_enterprise.proxy.common_utils.managed_object_attribution import ( ManagedObjectCreatorAttribution, ManagedObjectRow, + row_metadata, ) import litellm @@ -90,11 +91,6 @@ def _team_table(prisma_client: "PrismaClient") -> "TableActions[prisma_models.Li return TeamRepository(prisma_client).table -def _row_metadata(row: object) -> dict[str, object]: - metadata: Final = getattr(row, "metadata", None) - return cast(dict[str, object], metadata) if isinstance(metadata, dict) else {} # cast-ok: prisma Json object column - - class CheckBatchCost: def __init__( self, @@ -1063,7 +1059,9 @@ class CheckBatchCost: if (has_key_row and key_row is None) or (team_id and team_row is None): return True settings: Final = get_dynamic_logging_metadata( - UserAPIKeyAuth(metadata=_row_metadata(key_row), team_metadata=_row_metadata(team_row), team_id=team_id), + UserAPIKeyAuth( + metadata=dict(row_metadata(key_row)), team_metadata=dict(row_metadata(team_row)), team_id=team_id + ), proxy_config, ) callback_vars: Final = (settings.callback_vars if settings is not None else None) or {} diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 1fa461d7ab7..07048cf2c2f 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -7,6 +7,12 @@ attribution, priced against the deployment that served it. The poll's status che with BACKGROUND_RESPONSE_STATUS_POLL_CALL_ORIGIN and client reads are marked managed by the managed files hook, so both stay free. The spend event carries the id the client holds, the row's unified_object_id, so its spend log is found under that id with the _background_cost suffix. +The spend event reaches the creating key's or team's own logging callbacks the way that creator's +requests do. Its message content is redacted when the creator's key or team logging turns message +logging off, else as the proxy is set, the way the batch poll decides it. The row is claimed before +the creator is read, so reads that lose the race skip the lookups. A creator whose lookup fails, or +whose logging settings the logger rejects, is still billed, through the proxy's own callbacks only +and with its message content redacted. A read that finds no row, because the create could not store one, stores it under the reading key and bills it the same way, so a response the create left untracked is still billed once. A streamed background create is free on its own; its stream stores the row under the creating key and bills @@ -29,6 +35,7 @@ from typing import TYPE_CHECKING, Final, cast from litellm_enterprise.proxy.common_utils.managed_object_attribution import ( ManagedObjectCreatorAttribution, ManagedObjectRow, + ResponseCreator, ) from pydantic import BaseModel, ConfigDict, TypeAdapter @@ -54,11 +61,13 @@ from litellm.types.utils import ( if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.proxy.proxy_server import ProxyConfig from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.prisma_protocols import TableActions from litellm.router import Router _STORED_RESPONSE_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) +_CALLBACK_LIST_ADAPTER: Final[TypeAdapter[tuple[object, ...]]] = TypeAdapter(tuple[object, ...]) class _RoutedModelInfo(BaseModel): @@ -73,6 +82,8 @@ class _RoutedDeployment(BaseModel): _ROUTED_DEPLOYMENTS: Final = TypeAdapter(tuple[_RoutedDeployment, ...]) _RETRIEVE_RESPONSE_CALL_TYPE: Final = "aget_responses" +_CALLBACK_LIST_FIELDS: Final = frozenset({"success_callback", "failure_callback", "litellm_disabled_callbacks"}) +_REJECTED_CREATOR_LOGGING: Final[Mapping[str, object]] = MappingProxyType({"turn_off_message_logging": True}) @dataclass(frozen=True, slots=True) @@ -89,6 +100,7 @@ class BilledBackgroundResponse: custom_llm_provider: str | None litellm_params: Mapping[str, object] metadata: Mapping[str, object] + creator_logging: Mapping[str, object] @dataclass(frozen=True, slots=True) @@ -107,11 +119,51 @@ def _without_read_price(response: ResponsesAPIResponse) -> ResponsesAPIResponse: return unpriced -async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge: - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging - from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +def _string_callbacks(callbacks: object) -> tuple[str, ...]: + if not isinstance(callbacks, list): + return () + return tuple( + callback for callback in _CALLBACK_LIST_ADAPTER.validate_python(callbacks) if isinstance(callback, str) + ) - logging_obj: Final = LiteLLMLogging( + +def _running_proxy_config() -> "ProxyConfig": + from litellm.proxy.proxy_server import proxy_config + + return proxy_config + + +def creator_logging_fields(creator: ResponseCreator, proxy_config: "ProxyConfig | None") -> Mapping[str, object]: + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import key_and_team_logging_fields + + if not creator.settings_read: + return _REJECTED_CREATOR_LOGGING + try: + return key_and_team_logging_fields( + user_api_key_dict=UserAPIKeyAuth( + metadata=dict(creator.key_metadata), + team_id=creator.team_id, + team_metadata=dict(creator.team_metadata), + ), + proxy_config=proxy_config if proxy_config is not None else _running_proxy_config(), + ) + except Exception as e: + verbose_proxy_logger.error( + f"CheckResponsesCost: could not read the creator's logging settings, so only the proxy's " + f"callbacks get the billed event: {e}" + ) + return _REJECTED_CREATOR_LOGGING + + +def _billed_logging_obj(billed: BilledBackgroundResponse, creator_logging: Mapping[str, object]) -> "LiteLLMLogging": + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging + from litellm.utils import split_dynamic_success_callbacks + + sync_callbacks, async_callbacks = split_dynamic_success_callbacks( + _string_callbacks(creator_logging.get("success_callback")) + ) + return LiteLLMLogging( model=billed.model, messages=[{"role": "user", "content": ""}], stream=False, @@ -119,7 +171,27 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi start_time=datetime.now(), litellm_call_id=str(uuid.uuid4()), function_id=str(uuid.uuid4()), + dynamic_success_callbacks=list(sync_callbacks), + dynamic_async_success_callbacks=list(async_callbacks) or None, + kwargs={key: value for key, value in creator_logging.items() if key not in _CALLBACK_LIST_FIELDS}, ) + + +def _creator_or_proxy_logging_obj(billed: BilledBackgroundResponse) -> "LiteLLMLogging": + try: + return _billed_logging_obj(billed, billed.creator_logging) + except Exception as e: + verbose_proxy_logger.error( + f"CheckResponsesCost: the creator's logging settings were rejected, so only the proxy's " + f"callbacks get the billed event: {e}" + ) + return _billed_logging_obj(billed, _REJECTED_CREATOR_LOGGING) + + +async def log_billed_background_response(billed: BilledBackgroundResponse) -> BilledCharge: + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + logging_obj: Final = _creator_or_proxy_logging_obj(billed) logging_obj.update_environment_variables( litellm_params={**billed.litellm_params, "metadata": dict(billed.metadata)}, optional_params={}, @@ -187,6 +259,7 @@ class CheckResponsesCost: prisma_client: "PrismaClient", llm_router: "Router", log_billed_response: BilledResponseLogger = log_billed_background_response, + proxy_config: "ProxyConfig | None" = None, ): from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.router import Router @@ -196,6 +269,7 @@ class CheckResponsesCost: self.llm_router: Router = llm_router self._creator_attribution: Final = ManagedObjectCreatorAttribution(prisma_client, "CheckResponsesCost") self._log_billed_response: Final = log_billed_response + self._proxy_config: Final = proxy_config def _resolve_deployment(self, response_id: str) -> Deployment | None: model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id) @@ -307,11 +381,12 @@ class CheckResponsesCost: data={"status": job.status or PENDING_RESPONSE_STATUSES[0]}, ) - async def _billed_response( + def _billed_response( self, job: ManagedObjectRow, target: _PollTarget, response: ResponsesAPIResponse, + creator: ResponseCreator, ) -> BilledBackgroundResponse: deployment: Final = target.deployment return BilledBackgroundResponse( @@ -323,13 +398,14 @@ class CheckResponsesCost: litellm_params=_deployment_litellm_params(deployment) if deployment is not None else MappingProxyType({}), metadata=MappingProxyType( { - **(await self._creator_attribution.response_metadata(job, job.unified_object_id)), + **creator.metadata, INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, **target.model_metadata, **(_deployment_metadata(deployment) if deployment is not None else {}), **self._requested_model_group(target, job.team_id), } ), + creator_logging=creator_logging_fields(creator, self._proxy_config), ) def _requested_model_group(self, target: _PollTarget, team_id: str | None) -> Mapping[str, object]: @@ -354,7 +430,8 @@ class CheckResponsesCost: verbose_proxy_logger.info(f"Response {job.unified_object_id} was already billed elsewhere, skipping") return None try: - charge: Final = await self._log_billed_response(await self._billed_response(job, target, response)) + creator: Final = await self._creator_attribution.response_creator(job, job.unified_object_id) + charge: Final = await self._log_billed_response(self._billed_response(job, target, response, creator)) except BaseException: await self._release_billing_claim(job) raise diff --git a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py index 2e2c07fd544..a4a40a7a3d7 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py @@ -10,12 +10,16 @@ from __future__ import annotations from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime -from typing import TYPE_CHECKING, Final, Protocol, cast +from typing import TYPE_CHECKING, Final, Protocol, TypeGuard, cast from pydantic import TypeAdapter from litellm._logging import verbose_proxy_logger from litellm.constants import CLI_SESSION_KEY_PREFIX +from litellm.repositories.table_repositories import ( + DeletedVerificationTokenRepository, + DeprecatedVerificationTokenRepository, +) from litellm.repositories.team_repository import TeamRepository from litellm.repositories.user_repository import UserRepository from litellm.repositories.verification_token_repository import VerificationTokenRepository @@ -70,6 +74,7 @@ class ManagedObjectRow(Protocol): _TAG_LIST_ADAPTER: Final[TypeAdapter[list[object]]] = TypeAdapter(list[object]) _KEY_OWNER_SCOPE_PREFIX: Final = "key:" +_MAX_ROTATION_HOPS: Final = 16 def _string_tags(request_tags: object) -> list[str] | None: @@ -96,6 +101,20 @@ def _team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteL return TeamRepository(prisma_client).table +def _rotated_token_table( + prisma_client: PrismaClient, +) -> TableActions[prisma_models.LiteLLM_DeprecatedVerificationToken]: + return DeprecatedVerificationTokenRepository(prisma_client).table + + +def _deleted_token_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedVerificationToken]: + return DeletedVerificationTokenRepository(prisma_client).table + + +def _deleted_team_table(prisma_client: PrismaClient) -> TableActions[prisma_models.LiteLLM_DeletedTeamTable]: + return TeamRepository(prisma_client).deleted_table + + @dataclass(frozen=True, slots=True) class _CreatorRows: key: prisma_models.LiteLLM_VerificationToken | None @@ -103,6 +122,15 @@ class _CreatorRows: user: prisma_models.LiteLLM_UserTable | None +@dataclass(frozen=True, slots=True) +class ResponseCreator: + metadata: Mapping[str, object] + key_metadata: Mapping[str, object] + team_id: str | None + team_metadata: Mapping[str, object] + settings_read: bool = True + + class ManagedObjectCreatorAttribution: """Rebuilds the spend-tracking metadata of whoever created a managed object, so the poll job's spend row is attributed like the creating request: key hash, team, organization, @@ -117,7 +145,12 @@ class ManagedObjectCreatorAttribution: A background response also carries its end user, the creating key's project and its alias, and the key, team, and user budgets and per-model budgets, the way the read that billed it before did, and never names a key-scoped creator as its user. A key that sets no budget of - its own takes its budget tier's, the same rule auth applies. + its own takes its budget tier's, the same rule auth applies. Its creator's key and team + metadata come along too, so the billed event reaches the key's and team's logging callbacks + like that read did. A rotated key's metadata is read from the live key its rotations lead + to, and a deleted key's or team's from its deleted row. A lookup that fails leaves the + creator's settings unread and the attribution to the rows that still load, so the + response still bills and still counts toward their budgets. """ def __init__(self, prisma_client: PrismaClient, poller_name: str) -> None: @@ -128,31 +161,70 @@ class ManagedObjectCreatorAttribution: rows: Final = await self._rows(job, object_id) return {"user_api_key_user_id": job.created_by, **_shared_metadata(job, rows)} - async def response_metadata(self, job: ManagedObjectRow, object_id: str) -> Mapping[str, object]: - rows: Final = await self._rows(job, object_id) - end_user_id: Final = getattr(job, "end_user_id", None) - return { - "user_api_key_user_id": _creator_user_id(job.created_by), - **_shared_metadata(job, rows), - **({"user_api_key_end_user_id": end_user_id} if end_user_id else {}), - "user_api_key_project_id": getattr(rows.key, "project_id", None), - "user_api_key_project_alias": getattr( - getattr(rows.key, "litellm_project_table", None), "project_alias", None + async def response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator: + try: + return await self._read_response_creator(job, object_id) + except Exception as e: + verbose_proxy_logger.error( + f"{self._poller_name}: could not read the creator of {object_id}, so it bills without their " + f"settings: {e}" + ) + return ResponseCreator( + metadata=_response_creator_metadata(job, await self._rows(job, object_id)), + key_metadata={}, + team_id=getattr(job, "team_id", None), + team_metadata={}, + settings_read=False, + ) + + async def _read_response_creator(self, job: ManagedObjectRow, object_id: str) -> ResponseCreator: + api_key: Final = getattr(job, "api_key", None) + team_id: Final = getattr(job, "team_id", None) + rows: Final = _CreatorRows( + key=await self._live_key_row(api_key, job.created_by), + team=await self._live_team_row(team_id), + user=await self._user_row(object_id, _creator_user_id(job.created_by)), + ) + return ResponseCreator( + metadata=_response_creator_metadata(job, rows), + key_metadata=row_metadata(rows.key) if rows.key is not None else await self._retired_key_metadata(api_key), + team_id=team_id, + team_metadata=( + row_metadata(rows.team) if rows.team is not None else await self._deleted_team_metadata(team_id) ), - **_key_budget_fields(rows.key), - **_budget_fields( - rows.team, - "user_api_key_team_spend", - "user_api_key_team_max_budget", - "user_api_key_team_model_max_budget", - ), - **_budget_fields( - rows.user, - "user_api_key_user_spend", - "user_api_key_user_max_budget", - "user_api_key_user_model_max_budget", - ), - } + ) + + async def _retired_key_metadata(self, api_key: str | None) -> Mapping[str, object]: + if not _names_a_key_row(api_key): + return {} + current_token: Final = await self._token_rotated_to(api_key, _MAX_ROTATION_HOPS) + current_key: Final = ( + await _token_table(self._prisma_client).find_unique(where={"token": current_token}) + if current_token != api_key + else None + ) + if current_key is not None: + return row_metadata(current_key) + return row_metadata( + await _deleted_token_table(self._prisma_client).find_first( + where={"token": current_token}, order={"deleted_at": "desc"} + ) + ) + + async def _token_rotated_to(self, token: str, hops_left: int) -> str: + rotation: Final = await _rotated_token_table(self._prisma_client).find_unique(where={"token": token}) + if rotation is None or hops_left == 0: + return token + return await self._token_rotated_to(rotation.active_token_id, hops_left - 1) + + async def _deleted_team_metadata(self, team_id: str | None) -> Mapping[str, object]: + if not team_id: + return {} + return row_metadata( + await _deleted_team_table(self._prisma_client).find_first( + where={"team_id": team_id}, order={"deleted_at": "desc"} + ) + ) async def _rows(self, job: ManagedObjectRow, object_id: str) -> _CreatorRows: return _CreatorRows( @@ -175,25 +247,57 @@ class ManagedObjectCreatorAttribution: async def _key_row( self, object_id: str, api_key: str | None, created_by: str | None ) -> prisma_models.LiteLLM_VerificationToken | None: - if not api_key or _is_cli_session_key(api_key, created_by): - return None try: - return await _token_table(self._prisma_client).find_unique( - where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True} - ) + return await self._live_key_row(api_key, created_by) except Exception as e: verbose_proxy_logger.error(f"{self._poller_name}: could not look up the creating key for {object_id}: {e}") return None - async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None: - if not team_id: + async def _live_key_row( + self, api_key: str | None, created_by: str | None + ) -> prisma_models.LiteLLM_VerificationToken | None: + if not api_key or _is_cli_session_key(api_key, created_by): return None + return await _token_table(self._prisma_client).find_unique( + where={"token": api_key}, include={"litellm_budget_table": True, "litellm_project_table": True} + ) + + async def _team_row(self, object_id: str, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None: try: - return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id}) + return await self._live_team_row(team_id) except Exception as e: verbose_proxy_logger.error(f"{self._poller_name}: could not look up team {team_id} for {object_id}: {e}") return None + async def _live_team_row(self, team_id: str | None) -> prisma_models.LiteLLM_TeamTable | None: + if not team_id: + return None + return await _team_table(self._prisma_client).find_unique(where={"team_id": team_id}) + + +def _response_creator_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]: + end_user_id: Final = getattr(job, "end_user_id", None) + return { + "user_api_key_user_id": _creator_user_id(job.created_by), + **_shared_metadata(job, rows), + **({"user_api_key_end_user_id": end_user_id} if end_user_id else {}), + "user_api_key_project_id": getattr(rows.key, "project_id", None), + "user_api_key_project_alias": getattr(getattr(rows.key, "litellm_project_table", None), "project_alias", None), + **_key_budget_fields(rows.key), + **_budget_fields( + rows.team, + "user_api_key_team_spend", + "user_api_key_team_max_budget", + "user_api_key_team_model_max_budget", + ), + **_budget_fields( + rows.user, + "user_api_key_user_spend", + "user_api_key_user_max_budget", + "user_api_key_user_model_max_budget", + ), + } + def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, object]: api_key: Final = getattr(job, "api_key", None) @@ -221,6 +325,19 @@ def _shared_metadata(job: ManagedObjectRow, rows: _CreatorRows) -> Mapping[str, } +def row_metadata(row: object) -> Mapping[str, object]: + metadata: Final = getattr(row, "metadata", None) + if not isinstance(metadata, dict): + return {} + return cast(Mapping[str, object], metadata) # cast-ok: prisma Json object column + + +def _names_a_key_row(api_key: object) -> TypeGuard[str]: + from litellm.litellm_core_utils.litellm_logging import is_valid_sha256_hash + + return isinstance(api_key, str) and is_valid_sha256_hash(api_key) + + def _is_cli_session_key(api_key: str, created_by: str | None) -> bool: return bool(created_by) and api_key == f"{CLI_SESSION_KEY_PREFIX}-{created_by}" diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index f8e6498502b..3ac270fcf2c 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -1218,6 +1218,32 @@ def get_dynamic_logging_metadata( return callback_settings_obj +def _callback_settings_fields(callback_settings_obj: TeamCallbackMetadata) -> Mapping[str, object]: + callback_vars: Final = callback_settings_obj.callback_vars + return { + "success_callback": callback_settings_obj.success_callback, + "failure_callback": callback_settings_obj.failure_callback, + **({**callback_vars, TRUSTED_CALLBACK_VARS_FIELD: callback_vars} if callback_vars is not None else {}), + } + + +def key_and_team_logging_fields(user_api_key_dict: UserAPIKeyAuth, proxy_config: ProxyConfig) -> Mapping[str, object]: + callback_settings_obj: Final = get_dynamic_logging_metadata( + user_api_key_dict=user_api_key_dict, proxy_config=proxy_config + ) + disabled_callbacks: Final = (user_api_key_dict.metadata or {}).get("litellm_disabled_callbacks") + return MappingProxyType( + { + **(_callback_settings_fields(callback_settings_obj) if callback_settings_obj is not None else {}), + **( + {"litellm_disabled_callbacks": disabled_callbacks} + if disabled_callbacks and isinstance(disabled_callbacks, list) + else {} + ), + } + ) + + _get_dynamic_logging_metadata: Final = get_dynamic_logging_metadata @@ -2841,27 +2867,7 @@ async def add_litellm_data_to_request( request=request, request_data=data, metadata_key=_metadata_variable_name ) - # Team Callbacks controls - callback_settings_obj: Final = get_dynamic_logging_metadata( - user_api_key_dict=user_api_key_dict, proxy_config=proxy_config - ) - if callback_settings_obj is not None: - data["success_callback"] = callback_settings_obj.success_callback - data["failure_callback"] = callback_settings_obj.failure_callback - - if callback_settings_obj.callback_vars is not None: - # unpack callback_vars in data - for k, v in callback_settings_obj.callback_vars.items(): - data[k] = v - # Callbacks that must not honour request-supplied credentials read this - # proxy-owned field instead of the raw request kwargs. - data[TRUSTED_CALLBACK_VARS_FIELD] = callback_settings_obj.callback_vars - - # Add disabled callbacks from key metadata - if user_api_key_dict.metadata and "litellm_disabled_callbacks" in user_api_key_dict.metadata: - disabled_callbacks: Final = user_api_key_dict.metadata["litellm_disabled_callbacks"] - if disabled_callbacks and isinstance(disabled_callbacks, list): - data["litellm_disabled_callbacks"] = disabled_callbacks + data.update(key_and_team_logging_fields(user_api_key_dict=user_api_key_dict, proxy_config=proxy_config)) # Guardrails from key/team metadata and policy engine await move_guardrails_to_metadata( diff --git a/litellm/utils.py b/litellm/utils.py index 4687e4fa27a..897138a940c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1580,6 +1580,22 @@ def _get_wrapper_timeout(kwargs: dict[str, object]) -> object: return kwargs.get("timeout") if stream_timeout is None else stream_timeout +def _runs_as_async_success_callback(callback: str | Callable[..., object] | CustomLogger) -> bool: + get_coroutine_checker: Final = litellm_utils.get_coroutine_checker + return get_coroutine_checker().is_async_callable(callback) or callback in ("dynamodb", "s3") + + +def split_dynamic_success_callbacks( + callbacks: Sequence[str | Callable[..., object] | CustomLogger], +) -> tuple[ + tuple[str | Callable[..., object] | CustomLogger, ...], tuple[str | Callable[..., object] | CustomLogger, ...] +]: + return ( + tuple(callback for callback in callbacks if not _runs_as_async_success_callback(callback)), + tuple(callback for callback in callbacks if _runs_as_async_success_callback(callback)), + ) + + def check_coroutine(value) -> bool: get_coroutine_checker: Final = litellm_utils.get_coroutine_checker return get_coroutine_checker().is_async_callable(value) diff --git a/tests/integration/spend/test_background_response_billed_once.py b/tests/integration/spend/test_background_response_billed_once.py index 6d481a98fd0..1b7fc975217 100644 --- a/tests/integration/spend/test_background_response_billed_once.py +++ b/tests/integration/spend/test_background_response_billed_once.py @@ -1,8 +1,10 @@ import asyncio +import base64 import json import uuid from collections.abc import Callable, Mapping from concurrent.futures import ThreadPoolExecutor +from itertools import chain from typing import Final import httpx @@ -10,8 +12,12 @@ import openai import pytest from integration._support.client import Gateway, Scenario, eventually, held, list_value, object_value, string_value from integration._support.database import read_rows, write_rows +from integration._support.otlp_sink import decode_spans from integration._support.upstream import delete_scenario, register_scenario +from integration._support.wire import Request, wire_server from integration.caching.response_cache_case import entry_key, redis_store +from integration.security._callback_traffic import langfuse_sink +from integration.security._sinks import Recorder from integration.spend._background_responses import ( ATTRIBUTION_COLUMNS, CONCURRENT_READS, @@ -60,7 +66,7 @@ from integration.spend._background_responses import ( ) from openai._legacy_response import LegacyAPIResponse from openai.types.responses import Response -from pydantic import JsonValue +from pydantic import JsonValue, TypeAdapter from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN @@ -823,3 +829,78 @@ def test_a_client_cannot_make_a_background_create_free_by_forging_the_managed_ma scenario.cleanups.callback(forget_row, response_id) assert tuple(read_response(gateway, key, response_id) for _ in range(2)) == free_reads("completed", 2) assert_charged_exactly(lambda: key_spend(key), PRICE) + + +LANGFUSE_PUBLIC_KEY: Final = "pk-lf-background" +LANGFUSE_SECRET_KEY: Final = "sk-lf-background" +LANGFUSE_TRACES_PATH: Final = "/api/public/otel/v1/traces" +LANGFUSE_COST: Final = "langfuse.observation.cost_details" +_LANGFUSE_COST_DETAILS: Final = TypeAdapter(dict[str, float]) + + +def _langfuse_callback(host: str) -> dict[str, JsonValue]: + return { + "callback_name": "langfuse", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": LANGFUSE_PUBLIC_KEY, + "langfuse_secret_key": LANGFUSE_SECRET_KEY, + "langfuse_host": host, + }, + } + + +def _key_with_its_own_logging(scenario: Scenario, langfuse_url: str) -> str: + return scenario.key(metadata={"logging": [_langfuse_callback(langfuse_url)]}) + + +def _key_in_a_team_with_logging(scenario: Scenario, langfuse_url: str) -> str: + team_id: Final = scenario.team() + added: Final = scenario.gateway.request("POST", f"/team/{team_id}/callback", _langfuse_callback(langfuse_url)) + assert added.status_code == 200, added.text + return scenario.key(team_id=team_id) + + +LOGGING_ON: Final[Mapping[str, Callable[[Scenario, str], str]]] = { + "the_key": _key_with_its_own_logging, + "the_team": _key_in_a_team_with_logging, +} + + +def _trace_exports(langfuse: Recorder) -> tuple[Request, ...]: + return tuple(request for request in langfuse.requests() if request.target.startswith(LANGFUSE_TRACES_PATH)) + + +def _exported_costs(langfuse: Recorder) -> tuple[float, ...]: + spans: Final = tuple( + chain.from_iterable( + decode_spans(export.body, export.headers.get("content-type", "")) for export in _trace_exports(langfuse) + ) + ) + return tuple( + _LANGFUSE_COST_DETAILS.validate_json(string_value(span["attributes"][LANGFUSE_COST]))["total"] + for span in spans + if LANGFUSE_COST in span["attributes"] + ) + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("logging_on", LOGGING_ON.keys()) +def test_the_billed_cost_reaches_the_langfuse_project_set_on_the_creating_key_or_team( + gateway: Gateway, logging_on: str +) -> None: + with wire_server(langfuse_sink) as wire, gateway.scenario() as scenario: + langfuse: Final = Recorder(wire) + key: Final = LOGGING_ON[logging_on](scenario, wire.url) + upstream: Final = scripted_upstream(scenario, background_routes("completed")) + response_id: Final = create_background_response(scenario, key, upstream.model) + reads: Final = tuple(read_response(gateway, key, response_id) for _ in range(READS)) + assert reads in (settling_reads("completed", READS), free_reads("completed", READS)) + exported: Final = eventually( + lambda: _exported_costs(langfuse), + lambda costs: pytest.approx(PRICE) in costs, + seconds=SPEND_LANDING_SECONDS, + ) + assert [cost for cost in exported if cost > 0] == [pytest.approx(PRICE)] + credentials: Final = base64.b64encode(f"{LANGFUSE_PUBLIC_KEY}:{LANGFUSE_SECRET_KEY}".encode()).decode() + assert {export.headers.get("authorization") for export in _trace_exports(langfuse)} == {f"Basic {credentials}"} diff --git a/tests/unit/integrations/test_responses_background_cost.py b/tests/unit/integrations/test_responses_background_cost.py index 1bf95e35c4d..7d266c94700 100644 --- a/tests/unit/integrations/test_responses_background_cost.py +++ b/tests/unit/integrations/test_responses_background_cost.py @@ -282,6 +282,12 @@ class TestCheckResponsesCost: client = MagicMock() client.db = MagicMock() client.db.litellm_managedobjecttable = MagicMock() + client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + client.db.litellm_deprecatedverificationtoken.find_unique = AsyncMock(return_value=None) + client.db.litellm_deletedverificationtoken.find_first = AsyncMock(return_value=None) + client.db.litellm_deletedteamtable.find_first = AsyncMock(return_value=None) return client @pytest.fixture diff --git a/tests/unit/proxy/common_utils/test_check_responses_cost.py b/tests/unit/proxy/common_utils/test_check_responses_cost.py index d78c3ef6185..b8d5ddef3d2 100644 --- a/tests/unit/proxy/common_utils/test_check_responses_cost.py +++ b/tests/unit/proxy/common_utils/test_check_responses_cost.py @@ -4,6 +4,8 @@ Unit tests for CheckResponsesCost class import asyncio import dataclasses +import hashlib +import json from collections.abc import Mapping from dataclasses import dataclass from datetime import datetime @@ -127,10 +129,21 @@ class _InMemoryManagedObjectTable: return len(matched) +def _prisma_with_no_creator_rows() -> MagicMock: + prisma_client: Final = MagicMock() + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_deprecatedverificationtoken.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_deletedverificationtoken.find_first = AsyncMock(return_value=None) + prisma_client.db.litellm_deletedteamtable.find_first = AsyncMock(return_value=None) + return prisma_client + + def _poller_over(table: _InMemoryManagedObjectTable, billed_logger: _RecordingBilledLogger): from litellm_enterprise.proxy.common_utils.check_responses_cost import CheckResponsesCost - prisma_client: Final = MagicMock() + prisma_client: Final = _prisma_with_no_creator_rows() prisma_client.db.litellm_managedobjecttable = table poller: Final = CheckResponsesCost( proxy_logging_obj=MagicMock(), @@ -174,8 +187,7 @@ class TestCheckResponsesCost: @pytest.fixture def mock_prisma_client(self): """Create a mock Prisma client""" - client = MagicMock() - client.db = MagicMock() + client = _prisma_with_no_creator_rows() client.db.litellm_managedobjecttable = MagicMock() return client @@ -1702,26 +1714,60 @@ class _PricedBilling: table: _InMemoryManagedObjectTable response_id: str cost: float | None + logging_obj: object + + +@dataclass(frozen=True) +class _PricedPoller: + poller: object + prisma_client: MagicMock + table: _InMemoryManagedObjectTable + response_id: str + + async def bill(self, read: ResponsesAPIResponse) -> object: + from litellm_enterprise.proxy.common_utils.check_responses_cost import CheckResponsesCost + + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + + assert isinstance(self.poller, CheckResponsesCost) + charge: Final = await self.poller.bill_if_finished(response_id=self.response_id, response=read) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) + return charge _PRICED_LITELLM_PARAMS: Final[Mapping[str, object]] = MappingProxyType( {"model": "openai/gpt-5.6", "api_key": "sk-unused", "input_cost_per_token": 0.001, "output_cost_per_token": 0.002} ) +_CREATING_KEY_HASH: Final = hashlib.sha256(b"sk-alice").hexdigest() -async def _bill_through_a_priced_deployment( - read: ResponsesAPIResponse, +def _team_alpha_when_its_rows_exist( + team_row: object, deleted_team_row: object, default_team_settings: tuple[Mapping[str, object], ...] +) -> str | None: + return "team-alpha" if team_row is not None or deleted_team_row is not None or default_team_settings else None + + +def _priced_poller( litellm_params: Mapping[str, object] = _PRICED_LITELLM_PARAMS, model_info: Mapping[str, object] = MappingProxyType({}), + key_row: object = None, + team_row: object = None, + key_lookup_error: Exception | None = None, + default_team_settings: tuple[Mapping[str, object], ...] = (), + api_key: str = _CREATING_KEY_HASH, + rotated_to: Mapping[str, object] = MappingProxyType({}), + deleted_key_row: object = None, + deleted_team_row: object = None, model_name: str = "priced-gpt", requested_model: str = "priced-gpt", model_group_alias: Mapping[str, str] = MappingProxyType({}), team_id: str | None = None, -) -> _PricedBilling: +) -> _PricedPoller: + from types import SimpleNamespace + import litellm from litellm_enterprise.proxy.common_utils.check_responses_cost import CheckResponsesCost - from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.responses.utils import ResponsesAPIRequestUtils response_id: Final = ResponsesAPIRequestUtils._build_responses_api_response_id( @@ -1733,17 +1779,31 @@ async def _bill_through_a_priced_deployment( id="job-priced", unified_object_id=response_id, status="queued", - api_key="hash-alice", + api_key=api_key, file_object={"model": requested_model}, - team_id=team_id, + team_id=team_id or _team_alpha_when_its_rows_exist(team_row, deleted_team_row, default_team_settings), ), ) ) - prisma_client: Final = MagicMock() + key_rows: Final = {api_key: key_row, **{token: row for token, row in rotated_to.items()}} + prisma_client: Final = _prisma_with_no_creator_rows() prisma_client.db.litellm_managedobjecttable = table prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) - prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) - prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + side_effect=key_lookup_error or (lambda where, **_: key_rows.get(where["token"])) + ) + rotation_chain: Final = (api_key, *rotated_to) + rotated_from: Final = dict(zip(rotation_chain, rotation_chain[1:])) + prisma_client.db.litellm_deprecatedverificationtoken.find_unique = AsyncMock( + side_effect=lambda where: ( + SimpleNamespace(token=where["token"], active_token_id=rotated_from[where["token"]]) + if where["token"] in rotated_from + else None + ) + ) + prisma_client.db.litellm_deletedverificationtoken.find_first = AsyncMock(return_value=deleted_key_row) + prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=team_row) + prisma_client.db.litellm_deletedteamtable.find_first = AsyncMock(return_value=deleted_team_row) router: Final = litellm.Router( model_list=[ { @@ -1754,11 +1814,71 @@ async def _bill_through_a_priced_deployment( ], model_group_alias=dict(model_group_alias), ) - charge: Final = await CheckResponsesCost( - proxy_logging_obj=MagicMock(), prisma_client=prisma_client, llm_router=router - ).bill_if_finished(response_id=response_id, response=read) - await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) - return _PricedBilling(table=table, response_id=response_id, cost=charge.cost if charge is not None else None) + return _PricedPoller( + poller=CheckResponsesCost( + proxy_logging_obj=MagicMock(), + prisma_client=prisma_client, + llm_router=router, + proxy_config=_proxy_config_with(default_team_settings), + ), + prisma_client=prisma_client, + table=table, + response_id=response_id, + ) + + +async def _bill_through_a_priced_deployment( + read: ResponsesAPIResponse, + litellm_params: Mapping[str, object] = _PRICED_LITELLM_PARAMS, + model_info: Mapping[str, object] = MappingProxyType({}), + key_row: object = None, + team_row: object = None, + key_lookup_error: Exception | None = None, + default_team_settings: tuple[Mapping[str, object], ...] = (), + api_key: str = _CREATING_KEY_HASH, + rotated_to: Mapping[str, object] = MappingProxyType({}), + deleted_key_row: object = None, + deleted_team_row: object = None, + model_name: str = "priced-gpt", + requested_model: str = "priced-gpt", + model_group_alias: Mapping[str, str] = MappingProxyType({}), + team_id: str | None = None, +) -> _PricedBilling: + priced: Final = _priced_poller( + litellm_params=litellm_params, + model_info=model_info, + key_row=key_row, + team_row=team_row, + key_lookup_error=key_lookup_error, + default_team_settings=default_team_settings, + api_key=api_key, + rotated_to=rotated_to, + deleted_key_row=deleted_key_row, + deleted_team_row=deleted_team_row, + model_name=model_name, + requested_model=requested_model, + model_group_alias=model_group_alias, + team_id=team_id, + ) + charge: Final = await priced.bill(read) + return _PricedBilling( + table=priced.table, + response_id=priced.response_id, + cost=getattr(charge, "cost", None), + logging_obj=getattr(charge, "logging_obj", None), + ) + + +def _proxy_config_with(default_team_settings: tuple[Mapping[str, object], ...]): + from litellm.proxy.proxy_server import ProxyConfig + + proxy_config: Final = ProxyConfig() + proxy_config.config = ( + {"litellm_settings": {"default_team_settings": [dict(settings) for settings in default_team_settings]}} + if default_team_settings + else {} + ) + return proxy_config @pytest.mark.asyncio @@ -1780,7 +1900,7 @@ async def test_the_billing_event_prices_the_fetched_usage_at_the_deployment_rate assert billing.cost == pytest.approx(_PRICED_COST) assert payload["model_id"] == "deployment-custom" assert payload["model_group"] == "priced-gpt" - assert payload["metadata"]["user_api_key_hash"] == "hash-alice" + assert payload["metadata"]["user_api_key_hash"] == _CREATING_KEY_HASH assert read.hidden_params == {"response_cost": 0.0} assert billing.table.statuses == {"job-priced": "completed"} @@ -1945,3 +2065,428 @@ async def test_the_billing_event_keys_its_spend_row_apart_from_the_read_it_bills assert billed_request_id != billing.response_id assert billed_request_id.startswith(billing.response_id) + +_CREATOR_LANGFUSE_VARS: Final = MappingProxyType( + { + "langfuse_public_key": "pk-lf-creator", + "langfuse_secret_key": "sk-lf-creator", + "langfuse_host": "http://127.0.0.1:9", + } +) + + +def _key_row(metadata: Mapping[str, object]) -> object: + from types import SimpleNamespace + + return SimpleNamespace(key_alias=None, organization_id=None, spend=0.0, max_budget=None, metadata=dict(metadata)) + + +def _team_row(metadata: Mapping[str, object]) -> object: + from types import SimpleNamespace + + return SimpleNamespace(team_alias=None, organization_id=None, spend=0.0, max_budget=None, metadata=dict(metadata)) + + +def _langfuse_callback_settings(callback_vars: Mapping[str, object]) -> Mapping[str, object]: + return {"callback_settings": {"success_callback": ["langfuse"], "callback_vars": dict(callback_vars)}} + + +@dataclass(frozen=True) +class _CreatorLogging: + key_row: object + team_row: object + default_team_settings: tuple[Mapping[str, object], ...] + + +def _creator_logging(logging_on: str) -> _CreatorLogging: + logging: Final = { + "logging": [ + {"callback_name": "langfuse", "callback_type": "success", "callback_vars": dict(_CREATOR_LANGFUSE_VARS)} + ] + } + team_metadata: Final = { + "team logging": logging, + "team callback_settings": _langfuse_callback_settings(_CREATOR_LANGFUSE_VARS), + } + return _CreatorLogging( + key_row=_key_row(logging if logging_on == "key logging" else {}), + team_row=_team_row(team_metadata.get(logging_on, {})), + default_team_settings=( + ({"team_id": "team-alpha", "success_callback": ["langfuse"], **_CREATOR_LANGFUSE_VARS},) + if logging_on == "config team settings" + else () + ), + ) + + +def _recording_langfuse(): + from litellm.integrations.custom_logger import CustomLogger + from litellm.integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement + + class _RecordingLangfuse(LangfusePromptManagement): + def __init__(self) -> None: + CustomLogger.__init__(self) + self.events: tuple[tuple[object, object], ...] = () + + async def async_log_success_event( + self, kwargs: dict[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + params: Final = kwargs["standard_callback_dynamic_params"] + assert isinstance(params, Mapping) + self.events = (*self.events, (kwargs.get("response_cost"), params.get("langfuse_public_key"))) + + return _RecordingLangfuse() + + +async def _log_a_request_from(creator: _CreatorLogging) -> None: + import litellm + from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.litellm_pre_call_utils import key_and_team_logging_fields + + await litellm.acompletion( + model="openai/gpt-5.6", + messages=[{"role": "user", "content": "hi"}], + mock_response="hello", + **key_and_team_logging_fields( + user_api_key_dict=UserAPIKeyAuth( + metadata=getattr(creator.key_row, "metadata"), + team_id="team-alpha", + team_metadata=getattr(creator.team_row, "metadata"), + ), + proxy_config=_proxy_config_with(creator.default_team_settings), + ), + ) + await asyncio.sleep(0) + await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("proxy_logs_to_langfuse", [False, True]) +@pytest.mark.parametrize( + "logging_on", ["key logging", "team logging", "team callback_settings", "config team settings"] +) +async def test_the_billing_event_reaches_the_langfuse_projects_the_creators_own_requests_log_to( + monkeypatch: pytest.MonkeyPatch, logging_on: str, proxy_logs_to_langfuse: bool +): + """A key or team that logs to its own Langfuse project must see the billed charge in the same projects + its own requests log to, wherever that logging is configured and whether or not the proxy logs to + Langfuse too, or the charge only shows in the proxy's spend logs.""" + import litellm + from litellm.litellm_core_utils import litellm_logging + + langfuse: Final = _recording_langfuse() + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [langfuse]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", [langfuse] if proxy_logs_to_langfuse else []) + creator: Final = _creator_logging(logging_on) + + await _log_a_request_from(creator) + request_projects: Final = tuple(public_key for _, public_key in langfuse.events) + await _bill_through_a_priced_deployment( + _terminal_response(), + key_row=creator.key_row, + team_row=creator.team_row, + default_team_settings=creator.default_team_settings, + ) + billed_events: Final = langfuse.events[len(request_projects) :] + + assert _CREATOR_LANGFUSE_VARS["langfuse_public_key"] in request_projects + assert tuple(public_key for _, public_key in billed_events) == request_projects + assert all(cost == pytest.approx(_PRICED_COST) for cost, _ in billed_events) + + +def _rotations_for(retired: str, key_logging: object) -> Mapping[str, object]: + rotated_once: Final = hashlib.sha256(b"sk-alice-rotated").hexdigest() + if retired == "rotated key": + return {rotated_once: key_logging} + if retired == "key rotated twice": + return {rotated_once: None, hashlib.sha256(b"sk-alice-rotated-again").hexdigest(): key_logging} + return {} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("retired", ["rotated key", "key rotated twice", "deleted key", "deleted team"]) +async def test_a_creator_rotated_or_deleted_before_the_bill_still_gets_the_charge_in_its_own_langfuse_project( + monkeypatch: pytest.MonkeyPatch, retired: str +): + """A key rotated or deleted, or a team deleted, while its background response ran logged that response's + create to its own project, so the charge follows the settings kept on the key it was rotated to or on + its deleted row instead of leaving that project for the proxy's callbacks alone.""" + import litellm + from litellm.litellm_core_utils import litellm_logging + + langfuse: Final = _recording_langfuse() + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [langfuse]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + key_logging: Final = _creator_logging("key logging").key_row + + await _bill_through_a_priced_deployment( + _terminal_response(), + rotated_to=_rotations_for(retired, key_logging), + deleted_key_row=key_logging if retired == "deleted key" else SimpleNamespace(metadata={}), + deleted_team_row=_creator_logging("team callback_settings").team_row if retired == "deleted team" else None, + ) + + assert langfuse.events == ((pytest.approx(_PRICED_COST), _CREATOR_LANGFUSE_VARS["langfuse_public_key"]),) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failing_lookup, live_key_logging, team_logging", + [ + ("litellm_verificationtoken.find_unique", "key logging", "none"), + ("litellm_teamtable.find_unique", "none", "team callback_settings"), + ("litellm_deprecatedverificationtoken.find_unique", "none", "none"), + ("litellm_deletedverificationtoken.find_first", "none", "none"), + ], + ids=["live key", "live team", "rotated key", "deleted key"], +) +async def test_a_creator_lookup_that_keeps_failing_still_bills_the_response_once_without_the_creators_settings( + monkeypatch: pytest.MonkeyPatch, failing_lookup: str, live_key_logging: str, team_logging: str +): + """A creator lookup that fails on every read must not leave the response unbilled. It bills once under the + key and team the managed row stores, through the proxy's callbacks only, with the text left out since the + creator's redaction setting could not be read.""" + import litellm + from litellm.litellm_core_utils import litellm_logging + + langfuse: Final = _recording_langfuse() + capture: Final = _spend_capture_logger() + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [langfuse]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", [capture]) + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + priced: Final = _priced_poller( + key_row=_creator_logging(live_key_logging).key_row if live_key_logging != "none" else None, + team_row=_creator_logging(team_logging).team_row if team_logging != "none" else None, + ) + table_name, method = failing_lookup.split(".") + setattr( + getattr(priced.prisma_client.db, table_name), + method, + AsyncMock(side_effect=RuntimeError("database unavailable")), + ) + + charge: Final = await priced.bill(_read_with_text("the stored answer")) + await priced.bill(_read_with_text("the stored answer")) + + assert getattr(charge, "cost", None) == pytest.approx(_PRICED_COST) + assert priced.table.statuses == {"job-priced": "completed"} + assert langfuse.events == () + (payload,) = capture.payloads + assert payload["response_cost"] == pytest.approx(_PRICED_COST) + assert "the stored answer" not in json.dumps(payload, default=str) + metadata: Final = payload["metadata"] + assert isinstance(metadata, Mapping) + assert metadata["user_api_key_hash"] == _CREATING_KEY_HASH + assert metadata["user_api_key_team_id"] == ("team-alpha" if team_logging != "none" else None) + + +_TEAM_MODEL_BUDGET: Final = {"gpt-5": {"max_budget": 2.0, "budget_duration": "7d"}} +_LOADED_KEY_ATTRIBUTION: Final = {"user_api_key_alias": "prod-key", "user_api_key_max_budget": 5.0} +_LOADED_TEAM_ATTRIBUTION: Final = { + "user_api_key_team_alias": "Team Alpha", + "user_api_key_team_max_budget": 50.0, + "user_api_key_team_model_max_budget": _TEAM_MODEL_BUDGET, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failing_lookup, key_is_live, loaded_attribution", + [ + ("litellm_verificationtoken.find_unique", True, _LOADED_TEAM_ATTRIBUTION), + ("litellm_teamtable.find_unique", True, _LOADED_KEY_ATTRIBUTION), + ("litellm_deprecatedverificationtoken.find_unique", False, _LOADED_TEAM_ATTRIBUTION), + ("litellm_deletedverificationtoken.find_first", False, _LOADED_TEAM_ATTRIBUTION), + ], + ids=["live key", "live team", "rotated key", "deleted key"], +) +async def test_a_creator_lookup_that_fails_keeps_the_aliases_and_budgets_of_the_rows_that_did_load( + failing_lookup: str, key_is_live: bool, loaded_attribution: Mapping[str, object] +): + """One failed creator lookup must not strip what the other rows give the charge: the team's per-model budget + only counts a billed charge whose event carries it, and the aliases name who spent it.""" + table: Final = _InMemoryManagedObjectTable( + rows=( + _StoredResponse( + id="job-attributed", + unified_object_id="resp_attributed", + status="queued", + api_key=_CREATING_KEY_HASH, + team_id="team-alpha", + ), + ) + ) + billed_logger: Final = _RecordingBilledLogger() + poller: Final = _poller_over(table, billed_logger) + poller.prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=( + SimpleNamespace(key_alias="prod-key", organization_id=None, spend=4.0, max_budget=5.0, metadata={}) + if key_is_live + else None + ) + ) + poller.prisma_client.db.litellm_teamtable.find_unique = AsyncMock( + return_value=SimpleNamespace( + team_alias="Team Alpha", + organization_id=None, + spend=40.0, + max_budget=50.0, + model_max_budget=_TEAM_MODEL_BUDGET, + metadata={}, + ) + ) + table_name, method = failing_lookup.split(".") + setattr( + getattr(poller.prisma_client.db, table_name), + method, + AsyncMock(side_effect=RuntimeError("database unavailable")), + ) + + await poller.bill_if_finished(response_id="resp_attributed", response=_terminal_response()) + + assert table.statuses == {"job-attributed": "completed"} + (billed,) = billed_logger.billed + assert {field: billed.metadata.get(field) for field in loaded_attribution} == loaded_attribution + + +@pytest.mark.asyncio +async def test_a_team_logging_setting_the_logger_rejects_still_bills_the_response_once(monkeypatch: pytest.MonkeyPatch): + """A team whose stored callback settings the logger rejects, such as an os.environ reference it no + longer resolves, must not stop the bill: the charge reaches the proxy's callbacks and the response is + not put back for every later poll to fail on again.""" + import litellm + + capture: Final = _spend_capture_logger() + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", [capture]) + + billing: Final = await _bill_through_a_priced_deployment( + _terminal_response(), + key_row=_key_row({}), + team_row=_team_row(_langfuse_callback_settings({"langfuse_public_key": "os.environ/QA_UNSET_LANGFUSE_KEY"})), + ) + + (payload,) = capture.payloads + assert payload["response_cost"] == pytest.approx(_PRICED_COST) + assert billing.cost == pytest.approx(_PRICED_COST) + assert billing.table.statuses == {"job-priced": "completed"} + + +def _read_with_text(text: str) -> ResponsesAPIResponse: + return _terminal_response().model_copy( + update={ + "output": [ + { + "type": "message", + "id": "msg_stored", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + ] + } + ) + + +def _key_logging_to_langfuse(callback_vars: Mapping[str, object]) -> object: + return _key_row( + {"logging": [{"callback_name": "langfuse", "callback_type": "success", "callback_vars": dict(callback_vars)}]} + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("key_row", "deleted_key_row", "text_logged"), + [ + (_key_logging_to_langfuse(_CREATOR_LANGFUSE_VARS), None, True), + (_key_logging_to_langfuse({**_CREATOR_LANGFUSE_VARS, "turn_off_message_logging": "true"}), None, False), + (None, _key_logging_to_langfuse({**_CREATOR_LANGFUSE_VARS, "turn_off_message_logging": "true"}), False), + (None, None, True), + ], + ids=[ + "creator keeps message logging", + "creator turns message logging off", + "deleted creator turned message logging off", + "creator has no key row anywhere", + ], +) +async def test_the_billing_event_logs_the_response_text_only_as_the_creators_logging_settings_allow( + monkeypatch: pytest.MonkeyPatch, key_row: object, deleted_key_row: object, text_logged: bool +): + """The billed event follows the creating key's or team's message logging setting, a deleted key's included, + which falls back to the proxy's the way the batch poll decides it, so a key with no stored settings + anywhere logs the text exactly as the proxy is set.""" + import litellm + from litellm.litellm_core_utils import litellm_logging + + capture: Final = _spend_capture_logger() + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [_recording_langfuse()]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", [capture]) + monkeypatch.setattr(litellm, "turn_off_message_logging", False) + + await _bill_through_a_priced_deployment( + _read_with_text("the stored answer"), key_row=key_row, deleted_key_row=deleted_key_row + ) + + (payload,) = capture.payloads + assert payload["response_cost"] == pytest.approx(_PRICED_COST) + assert ("the stored answer" in json.dumps(payload, default=str)) is text_logged + + +@pytest.mark.asyncio +@pytest.mark.parametrize("caller_token", ["oauth2-raw-access-token", "sk-custom-auth-1234"]) +async def test_a_response_created_without_a_virtual_key_bills_into_its_teams_langfuse_project( + monkeypatch: pytest.MonkeyPatch, caller_token: str +): + """Under OAuth2 the row keeps the caller's raw token, and under custom auth the hash of an sk- token that + has no key row, so the charge follows the team's logging settings like that caller's own requests.""" + from litellm.proxy._types import UserAPIKeyAuth + + import litellm + from litellm.litellm_core_utils import litellm_logging + + langfuse: Final = _recording_langfuse() + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [langfuse]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + + await _bill_through_a_priced_deployment( + _terminal_response(), + api_key=UserAPIKeyAuth(api_key=caller_token).api_key or caller_token, + team_row=_creator_logging("team callback_settings").team_row, + ) + + assert langfuse.events == ((pytest.approx(_PRICED_COST), _CREATOR_LANGFUSE_VARS["langfuse_public_key"]),) + + +@pytest.mark.asyncio +async def test_a_key_that_disabled_a_callback_has_its_charge_logged_wherever_its_own_requests_are( + monkeypatch: pytest.MonkeyPatch, +): + """A key's stored list of disabled callbacks must treat the billed charge like that key's own requests, or + a project that shows the key's requests at $0 never shows the charge they add up to.""" + import litellm + from litellm.litellm_core_utils import litellm_logging + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "premium_user", True) + langfuse: Final = _recording_langfuse() + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", [langfuse]) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", [langfuse]) + creator: Final = _CreatorLogging( + key_row=_key_row({"litellm_disabled_callbacks": ["langfuse"]}), team_row=_team_row({}), default_team_settings=() + ) + + await _log_a_request_from(creator) + request_events: Final = langfuse.events + await _bill_through_a_priced_deployment(_terminal_response(), key_row=creator.key_row) + billed_events: Final = langfuse.events[len(request_events) :] + + assert len(billed_events) == len(request_events) + assert all(cost == pytest.approx(_PRICED_COST) for cost, _ in billed_events) diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index 32b77c3b564..abc1fe88c7a 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -1079,6 +1079,9 @@ def _managed_files_over(table: _UniqueManagedObjectTable) -> tuple[MagicMock, Ma prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=None) prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock(return_value=None) prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_deprecatedverificationtoken.find_unique = AsyncMock(return_value=None) + prisma_client.db.litellm_deletedverificationtoken.find_first = AsyncMock(return_value=None) + prisma_client.db.litellm_deletedteamtable.find_first = AsyncMock(return_value=None) proxy_logging: Final = MagicMock() proxy_logging.get_proxy_hook = MagicMock( return_value=_PROXY_LiteLLMManagedFiles(internal_usage_cache=DualCache(), prisma_client=prisma_client) From f3e1307ed94ec23024513642d1bd84fb87285339 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 23:26:52 +0000 Subject: [PATCH 022/235] fix(vertex_ai): log Gemini 3 sampling deprecation once per model per worker (#40868) * fix(vertex_ai): log Gemini 3 sampling deprecation once per process Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(vertex_ai): bound the Gemini 3 sampling warn-once cache and test it by behavior * fix(vertex_ai): log the Gemini 3 sampling deprecation once even when first calls race Pin the warned-model cap to its own constant so the shared LRU size knob cannot turn the dedupe off, and serialize the cached call so concurrent first requests for one model log it once * fix(vertex_ai): stop logging the Gemini 3 sampling deprecation for new models once 64 are remembered * fix(vertex_ai): give each mapped Gemini 3 model its own sampling notice and share one for unmapped names * fix(vertex_ai): key the Gemini 3 sampling notice on the loaded catalog so runtime-priced names share one notice --------- Co-authored-by: shivam Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../llms/vertex_ai/gemini/transformation.py | 2 +- .../vertex_and_google_ai_studio_gemini.py | 51 +++--- .../vertex_ai/gemini/test_transformation.py | 16 ++ ...test_vertex_and_google_ai_studio_gemini.py | 156 +++++++++++++++++- 4 files changed, 203 insertions(+), 22 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 7b832b321da..b6517a90250 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -868,7 +868,7 @@ def gemini_convert_messages_with_history( """ has_text_in_content = check_text_in_content(user_content) if has_text_in_content is False: - verbose_logger.warning( + verbose_logger.debug( "No text in user content. Adding a blank text to user content, to ensure Gemini doesn't fail the request. Relevant Issue - https://github.com/BerriAI/litellm/issues/5515" ) user_content.append( diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index ec40cdfd11f..a65ae9da3e9 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -3,10 +3,11 @@ ## Initial implementation - covers gemini + image gen calls import json import re +import threading import time from collections.abc import Awaitable, Callable, Mapping, Sequence from copy import deepcopy -from functools import partial +from functools import lru_cache, partial from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args @@ -28,6 +29,7 @@ from litellm.constants import ( ) from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.prompt_templates.factory import ( encode_tool_call_id_with_signature, @@ -196,6 +198,29 @@ def _unsupported_reasoning_effort(reasoning_effort: str) -> UnsupportedParamsErr ) +_SAMPLING_DEPRECATION_UNMAPPED_MODELS: Final = "models outside litellm's model map" +_sampling_deprecation_lock: Final = threading.Lock() + + +@lru_cache(maxsize=None) +def _log_gemini_sampling_params_deprecated(model: str) -> None: + verbose_logger.warning( + "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", + model, + ) + + +def _sampling_deprecation_notice_key(model: str) -> str: + catalog: Final = GetModelCostMap.loaded_model_cost_map() + is_cataloged: Final = any(name in catalog for name in (model, f"vertex_ai/{model}", f"gemini/{model}")) + return model if is_cataloged else _SAMPLING_DEPRECATION_UNMAPPED_MODELS + + +def _warn_gemini_sampling_params_deprecated(model: str) -> None: + with _sampling_deprecation_lock: + _log_gemini_sampling_params_deprecated(_sampling_deprecation_notice_key(model)) + + def _cache_creation_tokens(logging_obj: LoggingClass) -> int: recorded: Final[object] = getattr(logging_obj, "vertex_ai_cache_creation_input_tokens", 0) return recorded if isinstance(recorded, int) and recorded > 0 else 0 @@ -1266,7 +1291,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): allow_integer_enums: bool = True, ) -> dict: self._apply_include_server_side_tool_invocations(non_default_params, optional_params) - gemini_sampling_params_warned: bool = False for param, value in non_default_params.items(): if param == "temperature": if VertexGeminiConfig._is_gemini_3_or_newer(model): @@ -1275,28 +1299,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): "Warning: Setting temperature < 1.0 for Gemini 3 models (%s) can cause infinite loops, degraded reasoning performance, and failure on complex tasks. Strongly recommended to use temperature = 1.0 (default).", model, ) - if not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + _warn_gemini_sampling_params_deprecated(model) optional_params["temperature"] = value elif param == "top_p": - if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + if VertexGeminiConfig._is_gemini_3_or_newer(model): + _warn_gemini_sampling_params_deprecated(model) optional_params["top_p"] = value elif param == "top_k": - if VertexGeminiConfig._is_gemini_3_or_newer(model) and not gemini_sampling_params_warned: - verbose_logger.warning( - "DeprecationWarning: `temperature`, `top_p`, and `top_k` continue to function for Gemini 3+ (%s) but are planned for removal in a future release. Move sampling guidance into the `system` instructions instead.", - model, - ) - gemini_sampling_params_warned = True + if VertexGeminiConfig._is_gemini_3_or_newer(model): + _warn_gemini_sampling_params_deprecated(model) optional_params["top_k"] = value elif ( param == "stream" and value is True diff --git a/tests/unit/llms/vertex_ai/gemini/test_transformation.py b/tests/unit/llms/vertex_ai/gemini/test_transformation.py index 2ea02b776ab..ac2d43f64a1 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini/test_transformation.py @@ -1,12 +1,15 @@ import json +import logging import uuid +from typing import Final from unittest.mock import Mock import httpx import pytest import litellm +from litellm._logging import verbose_logger from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.llms.vertex_ai.gemini import transformation from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( @@ -469,3 +472,16 @@ async def test_vertex_ai_async_transform_inlines_only_the_urls_gemini_cannot_fet {"file_data": {"mime_type": "application/pdf", "file_uri": files_api_pdf}}, ] assert sorted(async_only_image_fetch.fetched) == sorted([plain_http_png, extensionless_https]) + + +def test_image_only_user_content_logs_the_blank_text_fixup_at_debug(caplog: pytest.LogCaptureFixture) -> None: + messages: Final = [ + {"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}}]} + ] + + with caplog.at_level(logging.DEBUG, logger=verbose_logger.name): + contents: Final = transformation.gemini_convert_messages_with_history(messages) + + fixups: Final = tuple(r for r in caplog.records if "No text in user content" in r.getMessage()) + assert tuple(r.levelno for r in fixups) == (logging.DEBUG,) + assert contents[0]["parts"][-1] == {"text": " "} diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 636d4034ccc..0796e125817 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1,7 +1,9 @@ import asyncio, importlib, os, uuid import json +import logging import re -from collections.abc import Awaitable, Callable, Mapping +import threading +from collections.abc import Awaitable, Callable, Iterator, Mapping from copy import deepcopy from typing import Final, List, Literal, cast, get_args from unittest.mock import MagicMock, Mock, patch @@ -13,7 +15,9 @@ from pydantic import BaseModel import litellm from litellm import ModelResponse, completion, embedding, image_generation +from litellm._logging import verbose_logger from litellm.litellm_core_utils.exception_mapping_utils import exception_type +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.llms.anthropic.pass_through.messages import handler as anthropic_messages_handler from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig @@ -21,6 +25,7 @@ from litellm.llms.vertex_ai.common_utils import GEMINI_MAX_ENUM_CHARACTERS, Vert from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, + _log_gemini_sampling_params_deprecated, ) from litellm.types.llms.vertex_ai import GeminiFinishReason, UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage @@ -6690,6 +6695,155 @@ def test_supported_reasoning_efforts_still_map(model): assert "thinkingConfig" in result +@pytest.fixture +def fresh_sampling_deprecation_cache() -> Iterator[None]: + _log_gemini_sampling_params_deprecated.cache_clear() + yield + _log_gemini_sampling_params_deprecated.cache_clear() + + +def _mapped_gemini_3_models() -> tuple[str, ...]: + return tuple(sorted(name for name in GetModelCostMap.loaded_model_cost_map() if name.startswith("gemini-3"))) + + +def _deprecation_messages(caplog: pytest.LogCaptureFixture) -> tuple[str, ...]: + return tuple(r.getMessage() for r in caplog.records if "DeprecationWarning" in r.getMessage()) + + +@pytest.mark.parametrize(("param", "value"), (("temperature", 0.2), ("top_p", 0.9), ("top_k", 5))) +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_is_logged_once_per_model( + param: str, value: float, caplog: pytest.LogCaptureFixture +) -> None: + first, second = _mapped_gemini_3_models()[:2] + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + mapped: Final = tuple( + config.map_openai_params( + non_default_params={param: value}, optional_params={}, model=model, drop_params=True + ) + for model in (first, first, first, second) + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 2 + assert f"({first})" in deprecations[0] + assert f"({second})" in deprecations[1] + assert all(m[param] == value for m in mapped) + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_is_logged_once_when_first_calls_race(caplog: pytest.LogCaptureFixture) -> None: + model: Final = _mapped_gemini_3_models()[0] + + def map_sampling_params() -> None: + VertexGeminiConfig().map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + racing_call: Final = threading.Thread(target=map_sampling_params) + + def start_racing_call_while_first_warning_logs(record: logging.LogRecord) -> bool: + if "DeprecationWarning" in record.getMessage() and racing_call.ident is None: + racing_call.start() + racing_call.join(timeout=0.5) + return True + + verbose_logger.addFilter(start_racing_call_while_first_warning_logs) + try: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + map_sampling_params() + racing_call.join() + finally: + verbose_logger.removeFilter(start_racing_call_while_first_warning_logs) + + assert len(_deprecation_messages(caplog)) == 1 + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_for_unmapped_names_does_not_hide_mapped_models( + caplog: pytest.LogCaptureFixture, +) -> None: + unmapped: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(100)) + mapped: Final = _mapped_gemini_3_models()[0] + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + for model in (*unmapped, mapped) * 2: + config.map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 2 + assert all(name not in deprecations[0] for name in unmapped) + assert f"({mapped})" in deprecations[1] + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_names_models_cataloged_only_under_vertex_ai( + caplog: pytest.LogCaptureFixture, +) -> None: + catalog: Final = GetModelCostMap.loaded_model_cost_map() + vertex_only: Final = next( + name.removeprefix("vertex_ai/") + for name in catalog + if name.startswith("vertex_ai/gemini-3") + and name.removeprefix("vertex_ai/") not in catalog + and f"gemini/{name.removeprefix('vertex_ai/')}" not in catalog + ) + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + VertexGeminiConfig().map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=vertex_only, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 1 + assert f"({vertex_only})" in deprecations[0] + + +@pytest.mark.usefixtures("fresh_sampling_deprecation_cache") +def test_gemini_3_sampling_deprecation_for_names_priced_at_runtime_shares_one_notice( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + priced: Final = tuple(f"gemini-3.9-flash-{uuid.uuid4().hex}" for _ in range(20)) + for name in priced: + monkeypatch.setitem( + litellm.model_cost, # pyright: ignore[reportUnknownMemberType] # registry is dynamically typed + f"gemini/{name}", + {"input_cost_per_token": 1e-7, "output_cost_per_token": 1e-7, "litellm_provider": "gemini"}, + ) + config: Final = VertexGeminiConfig() + + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + for model in priced * 2: + config.map_openai_params( + non_default_params={"temperature": 0.2}, optional_params={}, model=model, drop_params=True + ) + + deprecations: Final = _deprecation_messages(caplog) + assert len(deprecations) == 1 + assert all(name not in deprecations[0] for name in priced) + + +@pytest.mark.parametrize(("param", "value"), (("temperature", 0.5), ("top_p", 0.9), ("top_k", 5))) +def test_gemini_2_5_sampling_params_log_no_deprecation( + param: str, value: float, caplog: pytest.LogCaptureFixture +) -> None: + with caplog.at_level(logging.WARNING, logger=verbose_logger.name): + mapped: Final = VertexGeminiConfig().map_openai_params( + non_default_params={param: value}, + optional_params={}, + model=f"gemini-2.5-flash-{uuid.uuid4().hex}", + drop_params=True, + ) + + assert mapped[param] == value + assert not any("DeprecationWarning" in r.getMessage() for r in caplog.records) + + def _generate_content_body() -> dict: return { "candidates": [ From 5dce13d2c24965111ea1178e992f701d922306b2 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 16:33:01 -0700 Subject: [PATCH 023/235] fix(proxy): attribute retry attempt spend rows to their deployment and keep them out of daily request counts (#47658) * fix(proxy): attribute retry attempt spend rows to their own deployment and keep them out of daily request counts Each -attempt- row now carries the model, provider, api_base and model group of the deployment that billed it (looked up by model_info.id through the proxy router, injected as deployment_lookup) instead of copying the final row's, and is stamped internal_call_origin=billed_retry_attempt so the daily spend tables keep its spend and tokens while counting zero api_requests and failed_requests for it * fix(proxy): declare DeploymentLookup as a TypeAlias and leave the test file's untouched lines as they were * fix(proxy): mark attempt rows with billed_retry_attempt instead of internal_call_origin so model usage keeps their spend Bugbot: an internal_call_origin stamp made build_model_usage_transaction drop the attempt rows from LiteLLM_DailyModelUsage. The rows now carry their own SpendLogsMetadata flag, which only the daily request counters read * fix(proxy): read the billed_retry_attempt flag through a typed helper so the basedpyright gate holds * fix(proxy): keep the typed litellm_params local in _deployment_provider so the basedpyright gate holds * fix(proxy): annotate the validated litellm_params as LiteLLM_Params in _deployment_provider * fix(proxy): keep billed retry attempts out of daily request errors and strip the flag from caller metadata Bugbot: _record_request_error counted every attempt row as a caller failure in LiteLLM_DailyRequestErrors. Veria: a caller could send metadata.billed_retry_attempt to zero its own request counts, so the key joins the untrusted metadata fields the proxy strips on the way in * fix(proxy): stamp billed_retry_attempt explicitly in the spend log metadata so the basedpyright gate holds --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/constants.py | 1 + litellm/proxy/_types.py | 1 + litellm/proxy/db/db_spend_update_writer.py | 16 ++- .../proxy/hooks/proxy_track_cost_callback.py | 71 ++++++++-- litellm/proxy/litellm_pre_call_utils.py | 2 + .../spend_tracking/spend_tracking_utils.py | 13 +- .../gcs_pubsub/spend_logs_payload.json | 2 +- .../hooks/test_proxy_track_cost_callback.py | 123 ++++++++++++++++++ .../test_spend_management_endpoints.py | 2 +- 9 files changed, 216 insertions(+), 15 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 5b99a0f73dd..1c1dfb9aa1c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1654,6 +1654,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags" VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata" ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens" INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin" +BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt" MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response" SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated" SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 97a2fc31fb0..54acf59ef80 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3731,6 +3731,7 @@ class SpendLogsMetadata(TypedDict): routing_decision: StandardLoggingRoutingDecision | None kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None] internal_call_origin: InternalCallOrigin | None + billed_retry_attempt: ReadOnly[NotRequired[bool | None]] litellm_roi_estimator: ReadOnly[NotRequired[bool | None]] guardrail_information: list[StandardLoggingGuardrailInformation] | None eval_information: Any | None diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index bb6ffe675c0..4e98823ba77 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -29,6 +29,7 @@ from litellm._internal_context import service_target, with_service_target from litellm._logging import verbose_proxy_logger from litellm.caching import RedisCache from litellm.constants import ( + BILLED_RETRY_ATTEMPT_METADATA_KEY, DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME, DB_SPEND_UPDATE_JOB_NAME, INTERNAL_CALL_ORIGIN_METADATA_KEY, @@ -312,6 +313,11 @@ def _spend_tables_left_to_send( return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None +def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool: + """An earlier billed retry attempt's own spend row, not a request the caller sent.""" + return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True + + def _timed_request_duration_ms( payload: dict | SpendLogsPayload, request_status: Literal["success", "failure"], @@ -2805,11 +2811,12 @@ class DBSpendUpdateWriter: if not date: return metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"]) + internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults request_error_accumulator.record( payload=payload, request_status=prisma_client.get_request_status(payload), # pyright: ignore[reportUnknownMemberType] # legacy payload union; this caller supplies a typed spend payload date=date.split("T")[0], - is_internal_call=bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)), # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults + is_internal_call=internal_origin or _is_billed_retry_attempt(metadata), ) async def _common_add_spend_log_transaction_to_daily_transaction( @@ -2874,6 +2881,7 @@ class DBSpendUpdateWriter: endpoint = ROUTE_ENDPOINT_MAPPING.get(call_type, None) is_internal_call: Final = bool(_metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) + caller_request: Final = not is_internal_call and not _is_billed_retry_attempt(_metadata) cache_read_input_tokens: Final = extract_cache_read_tokens(usage_obj) compression_saved_tokens: Final = extract_compression_saved_tokens(_metadata) savings_spend: Final = compute_savings_spend( @@ -2908,9 +2916,9 @@ class DBSpendUpdateWriter: # requests the caller made: counting them inflates request-volume # readers, and an auto-router savings figure computed on a shadow # duplicate credits savings for traffic no user sent. - api_requests=0 if is_internal_call else 1, - successful_requests=1 if not is_internal_call and request_status == "success" else 0, - failed_requests=1 if not is_internal_call and request_status != "success" else 0, + api_requests=1 if caller_request else 0, + successful_requests=1 if caller_request and request_status == "success" else 0, + failed_requests=1 if caller_request and request_status != "success" else 0, cache_read_input_tokens=cache_read_input_tokens, cache_creation_input_tokens=extract_cache_creation_tokens(usage_obj), compression_saved_tokens=compression_saved_tokens, diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 1f8b352fed9..994310c6619 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -5,14 +5,14 @@ from contextlib import contextmanager from contextvars import ContextVar from datetime import datetime, timezone from types import MappingProxyType -from typing import TYPE_CHECKING, Final, Protocol, cast +from typing import TYPE_CHECKING, Final, Protocol, TypeAlias, cast from pydantic import TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.batches.batch_utils import batch_cost_is_final -from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED +from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED, BILLED_RETRY_ATTEMPT_METADATA_KEY from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( budget_reservation_from_metadata, @@ -54,11 +54,13 @@ from litellm.proxy.spend_tracking.spend_log_error_logger import ( ) from litellm.proxy.spend_tracking.spend_tracking_utils import ( # noqa: F401 # legacy module exports _sanitize_error_information_for_spend_logs, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export + deployment_params_provider, get_request_model_access_groups, sanitize_error_information_for_spend_logs, should_store_prompts_and_responses_in_spend_logs, ) from litellm.proxy.utils import BilledFailedAttempt, ProxyUpdateSpend +from litellm.types.router import Deployment from litellm.types.utils import ( CallTypes, LiteLLMBatch, @@ -187,6 +189,14 @@ def _proxy_spend_writer() -> DBSpendUpdateWriter: return proxy_logging_obj.db_spend_update_writer +DeploymentLookup: TypeAlias = Callable[[str], Deployment | None] + + +def _proxy_deployment(model_id: str) -> Deployment | None: + router: Final = get_llm_router() + return None if router is None else router.get_deployment(model_id=model_id) + + def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledFailedAttempt, ...]: """The earlier billed attempts ``_failure_fields_to_lift`` stashed on the request, if any.""" attempts: Final = request_data.get("billed_failed_attempts") @@ -197,7 +207,10 @@ def _billed_failed_attempts(request_data: Mapping[str, object]) -> tuple[BilledF def _failure_spend_rows( - request_data: Mapping[str, object], final_cost: float, late_guardrails_snapshot: object + request_data: Mapping[str, object], + final_cost: float, + late_guardrails_snapshot: object, + deployment_lookup: DeploymentLookup, ) -> tuple[tuple[float, Mapping[str, object]], ...]: """One ``(response_cost, kwargs)`` per spend row a failure writes: the final attempt under the request id, then every earlier attempt the provider also billed under its own row.""" @@ -209,25 +222,59 @@ def _failure_spend_rows( return ( (final_cost, final_row), *( - (attempt.response_cost, _attempt_spend_row_kwargs(request_data, attempt, position)) + ( + attempt.response_cost, + _attempt_spend_row_kwargs( + request_data, attempt, position, _attempt_deployment(attempt, deployment_lookup) + ), + ) for position, attempt in enumerate(_billed_failed_attempts(request_data)) ), ) +def _attempt_deployment(attempt: BilledFailedAttempt, deployment_lookup: DeploymentLookup) -> Deployment | None: + return None if attempt.deployment_id is None else deployment_lookup(attempt.deployment_id) + + +def _deployment_attribution(deployment: Deployment) -> Mapping[str, object]: + """The spend-log attribution fields of the deployment that served an attempt: the model string it + was called with, its provider, its configured ``api_base`` and its model group.""" + return MappingProxyType( + { + "model": deployment.litellm_params.model, + "custom_llm_provider": deployment_params_provider(deployment.litellm_params), + "api_base": deployment.litellm_params.api_base, + "model_group": deployment.model_name, + } + ) + + +_NO_ATTRIBUTION: Final[Mapping[str, object]] = MappingProxyType({}) + + def _attempt_spend_row_kwargs( - request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int + request_data: Mapping[str, object], attempt: BilledFailedAttempt, position: int, deployment: Deployment | None ) -> Mapping[str, object]: """The final attempt's row kwargs re-keyed for one earlier billed attempt: its own ``request_id`` (``-attempt-``, since ``request_id`` is the spend-log primary key and the retry count restarts after a fallback hop), its estimated usage and cost, its retry - count under ``attempted_retries`` and the deployment that billed it under ``model_info.id``.""" + count under ``attempted_retries``, the deployment that billed it under ``model_info.id`` and, + when the router still knows that deployment, its model, provider, ``api_base`` and model group. + The row is stamped ``billed_retry_attempt`` so the daily spend tables keep its spend, tokens + and model usage without counting it as a request the caller sent.""" suffix: Final = f"-attempt-{position}" + attribution: Final = _NO_ATTRIBUTION if deployment is None else _deployment_attribution(deployment) raw_params: Final = cast("Mapping[str, object]", request_data.get("litellm_params") or {}) # cast-ok: request dict - litellm_params: Final = dict(raw_params) + litellm_params: Final = { + **raw_params, + **({} if deployment is None else {"api_base": attribution["api_base"]}), + } metadata: Final = { **cast("Mapping[str, object]", litellm_params.get("metadata") or {}), # cast-ok: request dict "attempted_retries": attempt.attempted_retries, + BILLED_RETRY_ATTEMPT_METADATA_KEY: True, + **({} if deployment is None else {"model_group": attribution["model_group"]}), } metadata.pop("previous_models", None) if attempt.deployment_id is not None: @@ -238,6 +285,7 @@ def _attempt_spend_row_kwargs( litellm_params["metadata"] = metadata row: Final = { **request_data, + **({} if deployment is None else {k: attribution[k] for k in ("model", "custom_llm_provider")}), "litellm_params": litellm_params, "combined_usage_object": attempt.usage, "response_cost": attempt.response_cost, @@ -250,7 +298,11 @@ def _attempt_spend_row_kwargs( if isinstance(standard_logging_object, dict): payload: Final = cast("Mapping[str, object]", standard_logging_object) # cast-ok: StandardLoggingPayload dict if isinstance(payload.get("id"), str): - row["standard_logging_object"] = {**payload, "id": f"{payload['id']}{suffix}"} + row["standard_logging_object"] = { + **payload, + "id": f"{payload['id']}{suffix}", + **({} if deployment is None else {**attribution, "model_id": attempt.deployment_id}), + } return row @@ -262,10 +314,12 @@ class _ProxyDBLogger(CustomLogger): turn_off_message_logging: bool = False, message_logging: bool = True, spend_writer: Callable[[], DBSpendUpdateWriter] = _proxy_spend_writer, + deployment_lookup: DeploymentLookup = _proxy_deployment, ) -> None: super().__init__(turn_off_message_logging=turn_off_message_logging, message_logging=message_logging) self.spend_event_producer = spend_event_producer self._spend_writer: Final = spend_writer + self._deployment_lookup: Final = deployment_lookup async def async_log_success_event( self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime @@ -439,6 +493,7 @@ class _ProxyDBLogger(CustomLogger): cast("Mapping[str, object]", request_data), # cast-ok: request dict, copied per row below recovered_response_cost, failure_snapshot_with_late_guardrails, + self._deployment_lookup, ), ) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 3ac270fcf2c..09f53cecb72 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -29,6 +29,7 @@ from litellm._service_logger import ServiceLogging from litellm._uuid import uuid from litellm.constants import ( BAGGAGE_MAPPED_TAGS_METADATA_KEY, + BILLED_RETRY_ATTEMPT_METADATA_KEY, CALLER_SESSION_ID_METADATA_KEY, CLIENT_OUTPUT_CEILING_METADATA_KEY, CONSUMED_REQUEST_TAGS_METADATA_KEY, @@ -426,6 +427,7 @@ _UNTRUSTED_METADATA_CONTROL_FIELDS: Final = ( CONSUMED_REQUEST_TAGS_METADATA_KEY, ROUTING_REQUEST_TAGS_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, + BILLED_RETRY_ATTEMPT_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, "standard_logging_object", "litellm_roi_estimator", diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 091fb5d1305..62aee788ed0 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ( + BILLED_RETRY_ATTEMPT_METADATA_KEY, CALLER_SESSION_ID_METADATA_KEY, CLI_SESSION_KEY_PREFIX, EMPTY_MAPPING, @@ -187,6 +188,7 @@ _STAMPED_METADATA_KEYS: Final = frozenset( "used_client_oauth_token", KUBERNETES_POD_ROUTING_KEY, "litellm_roi_estimator", + BILLED_RETRY_ATTEMPT_METADATA_KEY, "user_api_key_user_email", ) ) @@ -249,6 +251,7 @@ def _get_spend_logs_metadata( guardrail_information=None, internal_call_origin=None, litellm_roi_estimator=False, + billed_retry_attempt=False, eval_information=None, cold_storage_object_key=cold_storage_object_key, litellm_overhead_time_ms=None, @@ -270,6 +273,8 @@ def _get_spend_logs_metadata( ) verbose_proxy_logger.debug("getting payload for SpendLogs, available keys in metadata: %s", [*metadata]) + retry_key: Final = BILLED_RETRY_ATTEMPT_METADATA_KEY + billed_retry_attempt: Final = retry_key in metadata and metadata[retry_key] is True # Filter the metadata dictionary to include only the specified keys clean_metadata: Final = SpendLogsMetadata( **MappingProxyType( @@ -283,6 +288,7 @@ def _get_spend_logs_metadata( used_client_oauth_token=used_client_oauth_token, kubernetes_pod_routing=kubernetes_pod_routing, litellm_roi_estimator=metadata.get("litellm_roi_estimator") is True, + billed_retry_attempt=billed_retry_attempt, user_api_key_user_email=_stored_user_email(metadata), ) _raw_key: Final = clean_metadata.get("user_api_key") @@ -453,7 +459,12 @@ def _sl_attribution_fallback( def _deployment_provider(deployment: DeploymentTypedDict) -> str | None: - litellm_params: Final = LiteLLM_Params.model_validate(deployment["litellm_params"]) + litellm_params: Final[LiteLLM_Params] = LiteLLM_Params.model_validate(deployment["litellm_params"]) + return deployment_params_provider(litellm_params) + + +def deployment_params_provider(litellm_params: LiteLLM_Params) -> str | None: + """The provider a deployment's ``litellm_params`` authenticate against, None when its model cannot be resolved.""" if litellm.LiteLLMProxyChatConfig.should_use_litellm_proxy_by_default(litellm_params=litellm_params): return LlmProviders.LITELLM_PROXY.value declared: Final = declared_authenticating_provider(litellm_params.model, litellm_params.custom_llm_provider) diff --git a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json index cfdf707c977..6e5461c3b60 100644 --- a/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json +++ b/tests/unit/integrations/gcs_pubsub/spend_logs_payload.json @@ -11,7 +11,7 @@ "user": "", "team_id": "", "organization_id": "", - "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", + "metadata": "{\"actor_agent_id\": null, \"target_agent_id\": null, \"billing_agent_id\": null, \"agent_execution_mode\": null, \"verified_human_user_id\": null, \"used_client_oauth_token\": null, \"applied_guardrails\": [], \"attempted_fallbacks\": null, \"original_model_group\": null, \"batch_models\": null, \"batch_successful_requests\": null, \"batch_failed_requests\": null, \"mcp_tool_call_metadata\": null, \"vector_store_request_metadata\": null, \"routing_decision\": null, \"internal_call_origin\": null, \"litellm_roi_estimator\": false, \"billed_retry_attempt\": false, \"router_metadata\": null, \"autorouter_savings_estimate\": null, \"autorouter_baseline_observation\": null, \"azure_spillover\": null, \"guardrail_information\": null, \"compression_savings\": null, \"litellm_gateway_injected_cache\": null, \"usage_object\": {\"completion_tokens\": 20, \"prompt_tokens\": 10, \"total_tokens\": 30, \"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"model_map_information\": {\"model_map_key\": \"gpt-4o\", \"model_map_value\": {\"key\": \"gpt-4o\", \"max_tokens\": 16384, \"max_input_tokens\": 128000, \"max_output_tokens\": 16384, \"input_cost_per_token\": 2.5e-06, \"cache_creation_input_token_cost\": null, \"cache_read_input_token_cost\": 1.25e-06, \"input_cost_per_character\": null, \"input_cost_per_token_above_128k_tokens\": null, \"input_cost_per_token_above_200k_tokens\": null, \"input_cost_per_query\": null, \"input_cost_per_second\": null, \"input_cost_per_audio_token\": null, \"input_cost_per_token_batches\": 1.25e-06, \"output_cost_per_token_batches\": 5e-06, \"output_cost_per_token\": 1e-05, \"output_cost_per_audio_token\": null, \"output_cost_per_character\": null, \"output_cost_per_token_above_128k_tokens\": null, \"output_cost_per_character_above_128k_tokens\": null, \"output_cost_per_token_above_200k_tokens\": null, \"output_cost_per_second\": null, \"output_cost_per_image\": null, \"output_vector_size\": null, \"litellm_provider\": \"openai\", \"mode\": \"chat\", \"supports_system_messages\": true, \"supports_response_schema\": true, \"supports_vision\": true, \"supports_function_calling\": true, \"supports_tool_choice\": true, \"supports_assistant_prefill\": false, \"supports_prompt_caching\": true, \"supports_audio_input\": false, \"supports_audio_output\": false, \"supports_pdf_input\": false, \"supports_embedding_image_input\": false, \"supports_native_streaming\": null, \"supports_web_search\": true, \"supports_reasoning\": false, \"search_context_cost_per_query\": {\"search_context_size_low\": 0.03, \"search_context_size_medium\": 0.035, \"search_context_size_high\": 0.05}, \"tpm\": null, \"rpm\": null, \"supported_openai_params\": [\"frequency_penalty\", \"logit_bias\", \"logprobs\", \"top_logprobs\", \"max_tokens\", \"max_completion_tokens\", \"modalities\", \"prediction\", \"n\", \"presence_penalty\", \"seed\", \"stop\", \"stream\", \"stream_options\", \"temperature\", \"top_p\", \"tools\", \"tool_choice\", \"function_call\", \"functions\", \"max_retries\", \"extra_headers\", \"parallel_tool_calls\", \"audio\", \"response_format\", \"user\"]}}, \"additional_usage_values\": {\"completion_tokens_details\": null, \"prompt_tokens_details\": null}, \"user_api_key\": null, \"user_api_key_alias\": null, \"user_api_key_team_id\": null, \"user_api_key_project_id\": null, \"user_api_key_project_alias\": null, \"user_api_key_org_id\": null, \"user_api_key_user_id\": null, \"user_api_key_user_email\": null, \"user_api_key_team_alias\": null, \"spend_logs_metadata\": null, \"requester_ip_address\": null, \"user_agent\": null, \"status\": null, \"proxy_server_request\": null, \"error_information\": null, \"attempted_retries\": null, \"max_retries\": null, \"kubernetes_pod_routing\": null}", "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index ba1b5536a07..bd997f10322 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -16,6 +16,7 @@ from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.collector import SpendEventConsumer from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter +from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction from litellm.proxy.db.spend_log_tool_index import response_tool_call_names from litellm.proxy.hooks.proxy_track_cost_callback import ( _get_budget_reservation_from_metadata, @@ -29,6 +30,7 @@ from litellm.proxy.utils import ProxyUpdateSpend from litellm.proxy.spend_tracking.spend_event import SpendEventDecodeError, build_spend_event, decode_spend_event from litellm.proxy.spend_tracking.spend_event_producer import SpendEventProducer, UnixAddress from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload +from litellm.types.router import Deployment, LiteLLM_Params from litellm.types.utils import CallTypes, LiteLLMBatch, ModelResponse, PromptTokensDetailsWrapper, Usage @@ -154,6 +156,127 @@ async def test_each_earlier_billed_attempt_gets_its_own_spend_row(): assert row["team_id"] == "t1" +def _deployment(model_name: str, model_id: str, model: str, api_base: str | None) -> Deployment: + return Deployment( + model_name=model_name, + litellm_params=LiteLLM_Params(model=model, api_base=api_base), + model_info={"id": model_id}, + ) + + +_TIMED_OUT_AT: Final = datetime(2026, 10, 20, 21, 24, 48, tzinfo=timezone.utc) + + +async def _spend_log_payloads_of_a_retried_timeout_across_a_fallback() -> list[SpendLogsPayload]: + """The spend-log payloads a timed-out request writes when attempt 0 and 1 ran on the anthropic + deployment of group ``essay`` and the request then fell back to the openai deployment of group + ``essay-fallback``: the final row first, then one row per earlier billed attempt.""" + from litellm.proxy.utils import BilledFailedAttempt + + deployments = { + "dep-anthropic": _deployment("essay", "dep-anthropic", "anthropic/claude-sonnet-4-6", None), + "dep-openai": _deployment("essay-fallback", "dep-openai", "openai/gpt-4.1", "https://api.openai.com/v1"), + } + usage = litellm.Usage(prompt_tokens=24, completion_tokens=0, total_tokens=24) + request_data = { + "model": "anthropic/claude-sonnet-4-6", + "custom_llm_provider": "anthropic", + "litellm_call_id": "call-1", + "litellm_params": {"api_base": "https://api.anthropic.com/v1/messages"}, + "standard_logging_object": { + "id": "call-1", + "model": "anthropic/claude-sonnet-4-6", + "custom_llm_provider": "anthropic", + "api_base": "https://api.anthropic.com/v1/messages", + "model_id": "dep-anthropic", + "model_group": "essay", + "metadata": {}, + "hidden_params": {}, + "model_map_information": {}, + }, + "metadata": {"attempted_retries": 1, "model_group": "essay", "model_info": {"id": "dep-anthropic"}}, + "combined_usage_object": usage, + "response_cost": 7.2e-05, + "billed_failed_attempts": ( + BilledFailedAttempt(attempted_retries=1, deployment_id="dep-anthropic", usage=usage, response_cost=7.2e-05), + BilledFailedAttempt(attempted_retries=0, deployment_id="dep-openai", usage=usage, response_cost=4.8e-05), + ), + } + exception = litellm.Timeout("Request timed out.", model="gpt-4.1", llm_provider="openai") + with patch( + "litellm.proxy.db.db_spend_update_writer.DBSpendUpdateWriter.update_database", + new_callable=AsyncMock, + ) as mock_update_database: + await ProxyDBLogger(deployment_lookup=deployments.get).async_post_call_failure_hook( + request_data=request_data, + original_exception=exception, + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key", user_id="u1"), + ) + return [ + get_logging_payload( + kwargs=call.kwargs["kwargs"], + response_obj=exception, + start_time=_TIMED_OUT_AT, + end_time=_TIMED_OUT_AT + timedelta(seconds=11), + ) + for call in mock_update_database.call_args_list + ] + + +@pytest.mark.asyncio +async def test_attempt_rows_are_attributed_to_the_deployment_that_billed_them(): + """Regression since #47416: an earlier attempt that ran on another deployment was written with the + final row's model, provider and api_base. Each attempt row names the deployment it ran on.""" + final_row, attempt_0, attempt_1 = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback() + + assert (final_row["model"], final_row["custom_llm_provider"], final_row["model_id"]) == ( + "anthropic/claude-sonnet-4-6", + "anthropic", + "dep-anthropic", + ) + assert final_row["api_base"] == "https://api.anthropic.com/v1/messages" + assert (attempt_0["model"], attempt_0["custom_llm_provider"], attempt_0["model_id"], attempt_0["model_group"]) == ( + "anthropic/claude-sonnet-4-6", + "anthropic", + "dep-anthropic", + "essay", + ) + assert (attempt_1["model"], attempt_1["custom_llm_provider"], attempt_1["model_id"], attempt_1["model_group"]) == ( + "openai/gpt-4.1", + "openai", + "dep-openai", + "essay-fallback", + ) + assert attempt_1["api_base"] == "https://api.openai.com/v1" + assert attempt_1["request_id"] == "call-1-attempt-1" + assert attempt_1["spend"] == 4.8e-05 + + +@pytest.mark.asyncio +async def test_attempt_rows_add_spend_but_not_requests_to_daily_spend(): + """Regression since #47416: one caller request with N billed earlier attempts counted N+1 requests in + the daily spend tables. The attempt rows keep their spend, tokens and model usage and count zero requests.""" + payloads = await _spend_log_payloads_of_a_retried_timeout_across_a_fallback() + prisma = MagicMock() + prisma.get_request_status = MagicMock(return_value="failure") + + daily = [ + await DBSpendUpdateWriter()._common_add_spend_log_transaction_to_daily_transaction( + payload=payload, prisma_client=prisma, type="user" + ) + for payload in payloads + ] + + assert all(row is not None for row in daily) + assert [row["api_requests"] for row in daily if row is not None] == [1, 0, 0] + assert [row["failed_requests"] for row in daily if row is not None] == [1, 0, 0] + assert [row["spend"] for row in daily if row is not None] == [7.2e-05, 7.2e-05, 4.8e-05] + assert [row["prompt_tokens"] for row in daily if row is not None] == [24, 24, 24] + model_usage = [build_model_usage_transaction(payload) for payload in payloads] + assert all(usage is not None for usage in model_usage), "attempt spend must still reach daily model usage" + assert [usage.spend for usage in model_usage if usage is not None] == [7.2e-05, 7.2e-05, 4.8e-05] + + @pytest.mark.asyncio async def test_async_post_call_failure_hook_carries_guardrail_info_from_litellm_metadata(): """ diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index ed8ae2db8b1..23bbe521670 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -3725,7 +3725,7 @@ class TestSpendLogsPayload: "model": "gpt-4o", "user": "", "team_id": "", - "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', + "metadata": '{"user_api_key_user_email": null, "actor_agent_id": null, "target_agent_id": null, "billing_agent_id": null, "agent_execution_mode": null, "verified_human_user_id": null, "applied_guardrails": [], "attempted_fallbacks": null, "original_model_group": null, "batch_models": null, "batch_successful_requests": null, "batch_failed_requests": null, "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "routing_decision": null, "internal_call_origin": null, "guardrail_information": null, "compression_savings": null, "litellm_gateway_injected_cache": null, "router_metadata": null, "autorouter_savings_estimate": null, "autorouter_baseline_observation": null, "azure_spillover": null, "used_client_oauth_token": null, "litellm_roi_estimator": false, "billed_retry_attempt": false, "usage_object": {"completion_tokens": 20, "prompt_tokens": 10, "total_tokens": 30, "completion_tokens_details": null, "prompt_tokens_details": null}, "model_map_information": {"model_map_key": "gpt-4o", "model_map_value": {"key": "gpt-4o", "max_tokens": 16384, "max_input_tokens": 128000, "max_output_tokens": 16384, "input_cost_per_token": 2.5e-06, "cache_creation_input_token_cost": null, "cache_read_input_token_cost": 1.25e-06, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": 1.25e-06, "output_cost_per_token_batches": 5e-06, "output_cost_per_token": 1e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_reasoning_token": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "openai", "mode": "chat", "supports_system_messages": true, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": false, "supports_prompt_caching": true, "supports_audio_input": false, "supports_audio_output": false, "supports_pdf_input": false, "supports_embedding_image_input": false, "supports_native_streaming": null, "supports_web_search": true, "supports_reasoning": false, "search_context_cost_per_query": {"search_context_size_low": 0.03, "search_context_size_medium": 0.035, "search_context_size_high": 0.05}, "tpm": null, "rpm": null, "supported_openai_params": ["frequency_penalty", "logit_bias", "logprobs", "top_logprobs", "max_tokens", "max_completion_tokens", "modalities", "prediction", "n", "presence_penalty", "seed", "stop", "stream", "stream_options", "temperature", "top_p", "tools", "tool_choice", "function_call", "functions", "max_retries", "extra_headers", "parallel_tool_calls", "audio", "response_format", "user"]}}, "additional_usage_values": {"completion_tokens_details": null, "prompt_tokens_details": null}}', "cache_key": "Cache OFF", "spend": 0.00022500000000000002, "total_tokens": 30, From 1c6d55c7e87968ff02c47f0bebfd3b9e265b1153 Mon Sep 17 00:00:00 2001 From: michelligabriele Date: Wed, 21 Oct 2026 01:42:53 +0200 Subject: [PATCH 024/235] fix(proxy): show the newest health check per model group on the model hub (#47552) --- litellm/proxy/db/health_check_latest.py | 13 +++++- .../public_endpoints/public_endpoints.py | 10 ++--- .../public_endpoints/public_v1/model_hub.py | 5 ++- .../unit/proxy/db/test_health_check_latest.py | 35 ++++++++++++++++ .../public_v1/test_model_hub.py | 27 ++++++++++-- .../public_endpoints/test_public_endpoints.py | 41 +++++++++++++++++++ 6 files changed, 118 insertions(+), 13 deletions(-) diff --git a/litellm/proxy/db/health_check_latest.py b/litellm/proxy/db/health_check_latest.py index 7672ed89e9a..024be7603d6 100644 --- a/litellm/proxy/db/health_check_latest.py +++ b/litellm/proxy/db/health_check_latest.py @@ -10,8 +10,9 @@ worker on every call. ``SELECT DISTINCT ON`` keeps the transfer at one row per from __future__ import annotations import json -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence from datetime import datetime, timezone +from types import MappingProxyType from typing import TYPE_CHECKING, Final, Protocol from pydantic import ConfigDict, JsonValue, TypeAdapter, field_validator @@ -113,3 +114,13 @@ async def fetch_latest_health_checks_for_models( except Exception as query_err: # noqa: BLE001 # a paged model list must not fail on its health decoration verbose_proxy_logger.error("Error getting latest health checks for models: %s", query_err) return () + + +def latest_by_model_name(rows: Iterable[LatestHealthCheckRow]) -> Mapping[str, LatestHealthCheckRow]: + """The newest row per model name. + + The queries above return one row per (model_id, model_name), so a model group with several + deployments comes back as several rows, plus one without a model_id if it was ever checked by + name. They arrive in model_id order with NULL last, which says nothing about which check is newest. + """ + return MappingProxyType({row.model_name: row for row in sorted(rows, key=lambda row: row.checked_at)}) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 8903b22c120..6ced7827846 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.proxy._types import ( CommonProxyErrors, ) +from litellm.proxy.db.health_check_latest import latest_by_model_name from litellm.proxy.utils import get_custom_url from litellm.repositories.table_repositories import ClaudeCodePluginRepository from litellm.router_strategy.complexity_router.fuse_presets import FusePresetCatalog, get_fuse_presets @@ -295,13 +296,8 @@ async def public_model_hub(): if prisma_client is not None: try: latest_checks: Final = await prisma_client.get_all_latest_health_checks() - for check in latest_checks: - key = check.model_id if check.model_id else check.model_name - if key: - health_check_dict = convert_health_check_to_dict(check) - health_checks_map[key] = health_check_dict - if check.model_name: - health_checks_map[check.model_name] = health_check_dict + for model_name, check in latest_by_model_name(latest_checks).items(): + health_checks_map[model_name] = convert_health_check_to_dict(check) except Exception: pass diff --git a/litellm/proxy/public_endpoints/public_v1/model_hub.py b/litellm/proxy/public_endpoints/public_v1/model_hub.py index 2f5eeeb3d4b..cd634e490b6 100644 --- a/litellm/proxy/public_endpoints/public_v1/model_hub.py +++ b/litellm/proxy/public_endpoints/public_v1/model_hub.py @@ -12,6 +12,7 @@ import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import CommonProxyErrors, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.db.health_check_latest import latest_by_model_name from litellm.proxy.list_api.common import PROBLEM_TYPE_BASE, ManagementProblem from litellm.proxy.list_api.in_memory import Cells, InMemoryListExecutor from litellm.proxy.list_api.list_framework import ( @@ -59,12 +60,12 @@ class PrismaHealthSnapshotLookup: checks: Final = await self.prisma_client.get_latest_health_checks_for_models(model_groups) return MappingProxyType( { - check.model_name: HealthSnapshot( + model_group: HealthSnapshot( status=check.status, response_time_ms=check.response_time_ms, checked_at=check.checked_at.isoformat() if check.checked_at else None, ) - for check in checks + for model_group, check in latest_by_model_name(checks).items() } ) diff --git a/tests/unit/proxy/db/test_health_check_latest.py b/tests/unit/proxy/db/test_health_check_latest.py index 29063d7b060..2f27be4fe61 100644 --- a/tests/unit/proxy/db/test_health_check_latest.py +++ b/tests/unit/proxy/db/test_health_check_latest.py @@ -7,8 +7,10 @@ import pytest from litellm.proxy.db.health_check_latest import ( LATEST_HEALTH_CHECKS_FOR_MODELS_SQL, LATEST_HEALTH_CHECKS_SQL, + LatestHealthCheckRow, fetch_latest_health_checks, fetch_latest_health_checks_for_models, + latest_by_model_name, query_latest_health_checks, ) from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -129,3 +131,36 @@ async def test_the_latest_health_check_read_renders_a_postgres_select_span( ) -> None: assert await fetch_latest_health_checks(_prisma([])) == () assert await postgres_span_names() == ("postgres.select LiteLLM_HealthCheckTable",) + + +def _latest_row(**overrides) -> LatestHealthCheckRow: + return LatestHealthCheckRow.model_validate(_raw_row(**overrides)) + + +@pytest.mark.parametrize("reverse", [False, True], ids=["sql-order", "reversed"]) +def test_latest_by_model_name_keeps_each_groups_newest_row_whatever_its_model_id(reverse): + """model_id ASC puts a name probe's NULL row, or a deleted deployment's larger id, after the fresh rows.""" + fresh = _latest_row(health_check_id="hc-a", model_id="deployment-a", checked_at="2026-10-16T11:10:00+00:00") + deleted = _latest_row( + health_check_id="hc-z", model_id="deployment-z", status="unhealthy", checked_at="2026-09-04T11:00:00+00:00" + ) + name_probe = _latest_row( + health_check_id="hc-null", model_id=None, status="unhealthy", checked_at="2026-09-04T11:20:04+00:00" + ) + other_group = _latest_row(health_check_id="hc-c", model_name="claude", model_id="deployment-c") + rows = (other_group, fresh, deleted, name_probe) + + latest = latest_by_model_name(rows[::-1] if reverse else rows) + + assert latest == {"gpt-4": fresh, "claude": other_group} + + +def test_latest_by_model_name_lets_a_newer_name_probe_speak_for_the_group(): + per_deployment = _latest_row( + health_check_id="hc-a", model_id="deployment-a", checked_at="2026-10-16T11:10:00+00:00" + ) + name_probe = _latest_row( + health_check_id="hc-null", model_id=None, status="unhealthy", checked_at="2026-10-16T12:00:00+00:00" + ) + + assert latest_by_model_name((per_deployment, name_probe))["gpt-4"] is name_probe diff --git a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py index 07886745222..43cdebd54b6 100644 --- a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py +++ b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py @@ -61,13 +61,19 @@ def _groups(response) -> list[str]: return [row["model_group"] for row in response.json()["data"]] -def _health_check(model_name: str, status: str = "healthy"): +def _health_check( + model_name: str, + status: str = "healthy", + *, + model_id: str | None = None, + checked_at: datetime = datetime(2026, 8, 1, 9, 30, tzinfo=timezone.utc), +): check = MagicMock() check.model_name = model_name - check.model_id = None + check.model_id = model_id check.status = status check.response_time_ms = 12.5 - check.checked_at = datetime(2026, 8, 1, 9, 30, tzinfo=timezone.utc) + check.checked_at = checked_at return check @@ -154,6 +160,21 @@ def test_the_latest_health_check_lands_on_its_row(monkeypatch): assert rows["model-000"]["health_status"] is None +def test_a_stale_row_sorting_after_the_fresh_one_does_not_speak_for_the_group(monkeypatch): + """The read orders by model_id with NULL last; that order used to pick the group's health.""" + fresh = _health_check( + "model-001", model_id="deployment-a", checked_at=datetime(2026, 10, 16, 11, 10, tzinfo=timezone.utc) + ) + stale = _health_check("model-001", status="unhealthy", checked_at=datetime(2026, 9, 4, 11, 20, tzinfo=timezone.utc)) + prisma_client, _ = _recording_prisma([fresh, stale]) + _publish(monkeypatch, _named(3), prisma_client=prisma_client) + + rows = {row["model_group"]: row for row in _get().json()["data"]} + + assert rows["model-001"]["health_status"] == "healthy" + assert rows["model-001"]["health_checked_at"] == "2026-10-16T11:10:00+00:00" + + def test_a_health_read_that_returns_nothing_still_serves_the_page(monkeypatch): prisma_client, read = _recording_prisma() read.return_value = [] diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index d5bacd9fd3c..8f27365fcc1 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -15,6 +15,7 @@ from pydantic import TypeAdapter import litellm from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.db.health_check_latest import LatestHealthCheckRow from litellm.proxy.public_endpoints import router from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_presets from litellm.types.proxy.management_endpoints.model_management_endpoints import ( @@ -852,6 +853,46 @@ def test_public_model_hub_mixed_health_statuses(): app.dependency_overrides.clear() +def _stored_health_check(model_id: str | None, status: str, checked_at: datetime) -> LatestHealthCheckRow: + return LatestHealthCheckRow( + health_check_id=f"hc-{model_id}", + model_name="gpt-4", + model_id=model_id, + status=status, + checked_at=checked_at, + created_at=checked_at, + updated_at=checked_at, + ) + + +def test_public_model_hub_reports_the_newest_check_of_a_group_not_the_last_row_read(): + """Rows come back in model_id order with NULL last, so a stale name-probe row used to overwrite the fresh one.""" + app = FastAPI() + app.include_router(router) + app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + client = TestClient(app) + fresh = _stored_health_check("deployment-a", "healthy", datetime(2026, 10, 16, 11, 10, tzinfo=timezone.utc)) + stale = _stored_health_check(None, "unhealthy", datetime(2026, 9, 4, 11, 20, tzinfo=timezone.utc)) + mock_prisma = MagicMock() + mock_prisma.get_all_latest_health_checks = AsyncMock(return_value=(fresh, stale)) + + with ( + patch("litellm.public_model_groups", ["gpt-4"]), + patch("litellm.proxy.proxy_server.get_model_group_info") as mock_get_info, + patch("litellm.proxy.proxy_server.llm_router", MagicMock()), + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + ): + mock_get_info.return_value = [ + ModelGroupInfoProxy(model_group="gpt-4", providers=["openai"], is_public_model_group=True) + ] + response = client.get("/public/model_hub", headers={"Authorization": "Bearer test-key"}) + + assert response.status_code == 200 + assert response.json()[0]["health_status"] == "healthy" + assert response.json()[0]["health_checked_at"] == "2026-10-16T11:10:00+00:00" + app.dependency_overrides.clear() + + # --------------------------------------------------------------------------- # /public/agent_hub # --------------------------------------------------------------------------- From c29a4f56a3006221706c30639c22e653a9db4a84 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Tue, 20 Oct 2026 16:51:55 -0700 Subject: [PATCH 025/235] fix(gemini)!: honor supports_sampling_params on the Gemini and Vertex path and flag Gemini 3.6 Flash, 3.5 Flash-Lite and later (#47645) * fix(gemini)!: honor supports_sampling_params on the Gemini and Vertex path and flag Gemini 3.6 Flash, 3.5 Flash-Lite and later Google ignores temperature, top_p and top_k on Gemini 3.6 Flash, Gemini 3.5 Flash-Lite and every later Gemini release, and says a later generation will return 400 for them. The model map now carries supports_sampling_params: false on those rows, and the Gemini and Vertex transformation reads the flag the way the Anthropic and Bedrock paths already do, dropping the params under drop_params and otherwise raising the client-side 400 that names drop_params BREAKING CHANGE: a request that sends temperature, top_p or top_k to a flagged Gemini model without drop_params now gets a 400 instead of a 200 whose sampling params Google silently ignored Fixes #47015 * fix(gemini): gate the passed-through top_k where the request body is built and keep allowed_openai_params working top_k is a provider-specific kwarg that never reaches the OpenAI param mapping, so the flag has to be applied while the Gemini request body is built, the same boundary Anthropic uses. The mapping now skips a flagged sampling param instead of raising, because the shared unsupported-param check already returns the 400 that names drop_params, and a param the caller lists in allowed_openai_params is forwarded verbatim as that 400 promises * fix(gemini): filter top_k out of the generation config instead of mutating optional_params * test(gemini): type the sampling-flag twin test and keep its locals out of the loop body * fix(vertex_ai): reject or drop sampling params on batch rows for flagged Gemini models The batch JSONL wrapper calls map_openai_params directly and skips get_optional_params, so a flagged model's temperature, top_p and top_k vanished from the uploaded row with no 400. Batch rows now get the same contract as a real-time call: a 400 naming drop_params, or the knobs dropped when litellm.drop_params is on * fix(vertex_ai): ignore null sampling knobs on batch rows for flagged Gemini models A batch row carrying temperature: null, top_p: null or top_k: null means no override, the same as on a real-time call, so the row is no longer refused with a 400 and the null knobs are simply not forwarded --- litellm/llms/gemini/chat/transformation.py | 4 +- .../llms/vertex_ai/files/transformation.py | 16 +- .../llms/vertex_ai/gemini/transformation.py | 14 +- .../vertex_and_google_ai_studio_gemini.py | 41 ++++- ...odel_prices_and_context_window_backup.json | 27 +++ litellm/types/utils.py | 1 + model_prices_and_context_window.json | 27 +++ .../test_vertex_ai_files_transformation.py | 57 ++++++ ...test_vertex_and_google_ai_studio_gemini.py | 165 +++++++++++++++++- .../test_model_prices_and_context_window.py | 21 +++ .../unit/test_router_model_cost_isolation.py | 2 +- 11 files changed, 361 insertions(+), 14 deletions(-) diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index 143652ee77c..936919af83d 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -82,9 +82,9 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): return "tts" in model def get_supported_openai_params(self, model: str) -> list[str]: + sampling_params: Final = ("temperature", "top_p") if self.supports_sampling_params(model) else () supported_params: Final = [ - "temperature", - "top_p", + *sampling_params, "max_tokens", "max_completion_tokens", "stream", diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index ea71e5fef50..176569f5d8d 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -1107,14 +1107,18 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): """ wrapper to call VertexGeminiConfig.map_openai_params """ - from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( - VertexGeminiConfig, - ) - config: Final = VertexGeminiConfig() - _model: Final = openai_request_body.get("model", "") + raw_model: Final = openai_request_body.get("model", "") + model: Final = raw_model if isinstance(raw_model, str) else "" + sampling_params: Final = tuple( + p for p in ("temperature", "top_p", "top_k") if openai_request_body.get(p) is not None + ) + if sampling_params and not VertexGeminiConfig.supports_sampling_params(model): + VertexGeminiConfig.reject_unsupported_sampling_param( + model=model, param=", ".join(sampling_params), drop_params=False + ) vertex_params: Final = config.map_openai_params( - model=_model, + model=model, non_default_params=openai_request_body, optional_params={}, drop_params=False, diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index b6517a90250..e5ce395031a 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -16,6 +16,7 @@ from pydantic import BaseModel, TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_image_mime_type_from_url, ) @@ -1189,7 +1190,18 @@ def transform_request_body( # labels: optional explicit param and/or metadata.requester_metadata (OpenAI metadata) labels: Final = pop_vertex_request_labels(optional_params, litellm_params) - filtered_params = {k: v for k, v in optional_params.items() if _get_equivalent_key(k, set(config_fields))} + from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig + + drops_top_k: Final = "top_k" in optional_params and not VertexGeminiConfig.supports_sampling_params(model) + if drops_top_k: + request_params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: untyped shared signature + VertexGeminiConfig.reject_unsupported_sampling_param( + model=model, + param="top_k", + drop_params=drops_unsupported_params(request_params.get("drop_params")), + ) + config_keys: Final = set(config_fields) - {"top_k"} if drops_top_k else set(config_fields) + filtered_params = {k: v for k, v in optional_params.items() if _get_equivalent_key(k, config_keys)} generation_config: Final[GenerationConfig | None] = GenerationConfig(**filtered_params) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index a65ae9da3e9..241b2d2a3a0 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -429,11 +429,43 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return False return True + @staticmethod + def supports_sampling_params(model: str) -> bool: + """Whether Google still honors ``temperature`` / ``top_p`` / ``top_k`` for ``model``. + + Driven by ``supports_sampling_params`` in the model map: Gemini 3.6 Flash, 3.5 Flash-Lite + and later accept the fields but ignore them, and a later generation will 400 on them. + A model the map says nothing about keeps forwarding them. ``temperature`` and ``top_p`` + leave ``get_supported_openai_params`` when the flag is off, so the shared unsupported-param + check raises or drops them and ``allowed_openai_params`` still forwards them verbatim; + ``top_k`` is a provider-specific kwarg that bypasses that check and is gated while the + request body is built.""" + rows: Final = cast(Mapping[str, Mapping[str, object]], litellm.model_cost) # cast-ok: JSON rows by model id + bare: Final = model.split("/")[-1] + for candidate in (model, bare, f"vertex_ai/{bare}", f"gemini/{bare}"): + if isinstance(flag := rows.get(candidate, {}).get("supports_sampling_params"), bool): + return flag + return True + + @staticmethod + def reject_unsupported_sampling_param(model: str, param: str, drop_params: bool) -> None: + """Return under ``drop_params`` so the caller drops ``param``, else raise the client-side 400 + that names the escape hatch.""" + if litellm.drop_params or drop_params: + return + raise UnsupportedParamsError( + message=( + f"{model} does not support {param}: the model map flags it `supports_sampling_params: false` " + "because Google ignores sampling params on this model. " + "To drop unsupported params, set `litellm.drop_params = True`." + ), + status_code=400, + ) + def get_supported_openai_params(self, model: str) -> list[str]: + sampling_params: Final = ("temperature", "top_p", "top_k") if self.supports_sampling_params(model) else () supported_params: Final = [ - "temperature", - "top_p", - "top_k", + *sampling_params, "max_tokens", "max_completion_tokens", "stream", @@ -1291,7 +1323,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): allow_integer_enums: bool = True, ) -> dict: self._apply_include_server_side_tool_invocations(non_default_params, optional_params) + supports_sampling_params: Final = self.supports_sampling_params(model) for param, value in non_default_params.items(): + if param in ("temperature", "top_p", "top_k") and not supports_sampling_params: + continue if param == "temperature": if VertexGeminiConfig._is_gemini_3_or_newer(model): if value is not None and value < 1.0: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 04a3f9225ce..98fee42de3c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21921,6 +21921,7 @@ ], "supports_function_calling": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true }, @@ -21945,6 +21946,7 @@ ], "supports_function_calling": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true }, @@ -21976,6 +21978,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true }, @@ -22039,6 +22042,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true }, @@ -29435,6 +29439,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -30157,6 +30162,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -30218,6 +30224,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -30286,6 +30293,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -30335,6 +30343,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": false, "supports_url_context": false, @@ -31761,6 +31770,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -31945,6 +31955,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32007,6 +32018,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32069,6 +32081,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32438,6 +32451,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32499,6 +32513,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32567,6 +32582,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32616,6 +32632,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": false, "supports_url_context": false, @@ -33210,6 +33227,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_vision": true }, "github_copilot/gemini-3.8-flash": { @@ -33228,6 +33246,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_vision": true }, "github_copilot/gpt-3.5-turbo": { @@ -52623,6 +52642,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -65841,6 +65861,7 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_vision": true, "source": "https://deepinfra.com/pricing", "supports_audio_input": true @@ -69282,6 +69303,7 @@ "perplexity/google/gemini-3.5-flash-lite": { "litellm_provider": "perplexity", "mode": "responses", + "supports_sampling_params": false, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 3e-07, @@ -69292,6 +69314,7 @@ "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", + "supports_sampling_params": false, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 1.5e-06, @@ -69302,6 +69325,7 @@ "perplexity/google/gemini-3.7-flash": { "litellm_provider": "perplexity", "mode": "responses", + "supports_sampling_params": false, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 7.5e-07, @@ -71393,6 +71417,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_vision": true, "supports_web_search": true }, @@ -71410,6 +71435,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_vision": true, "supports_web_search": true }, @@ -74340,6 +74366,7 @@ "perplexity/google/gemini-3.8-flash": { "litellm_provider": "perplexity", "mode": "responses", + "supports_sampling_params": false, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.75e-06, "cache_read_input_token_cost": 7.5e-08, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f86c6cfd4b6..5939a350970 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -212,6 +212,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_parallel_function_calling: bool | None supports_web_search: bool | None supports_reasoning: bool | None + supports_sampling_params: ReadOnly[bool | None] supports_adaptive_thinking: bool | None supports_legacy_thinking: ReadOnly[bool | None] thinking_always_on: ReadOnly[bool | None] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 04a3f9225ce..98fee42de3c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -21921,6 +21921,7 @@ ], "supports_function_calling": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true }, @@ -21945,6 +21946,7 @@ ], "supports_function_calling": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true }, @@ -21976,6 +21978,7 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true }, @@ -22039,6 +22042,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_tool_choice": true, "supports_vision": true }, @@ -29435,6 +29439,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -30157,6 +30162,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -30218,6 +30224,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -30286,6 +30293,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -30335,6 +30343,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": false, "supports_url_context": false, @@ -31761,6 +31770,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -31945,6 +31955,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32007,6 +32018,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32069,6 +32081,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32438,6 +32451,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32499,6 +32513,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32567,6 +32582,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -32616,6 +32632,7 @@ "supports_reasoning": true, "supports_minimal_reasoning_effort": false, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": false, "supports_url_context": false, @@ -33210,6 +33227,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_vision": true }, "github_copilot/gemini-3.8-flash": { @@ -33228,6 +33246,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_vision": true }, "github_copilot/gpt-3.5-turbo": { @@ -52623,6 +52642,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_system_messages": true, "supports_tool_choice": true, "supports_url_context": true, @@ -65841,6 +65861,7 @@ "supports_tool_choice": true, "supports_response_schema": true, "supports_reasoning": true, + "supports_sampling_params": false, "supports_vision": true, "source": "https://deepinfra.com/pricing", "supports_audio_input": true @@ -69282,6 +69303,7 @@ "perplexity/google/gemini-3.5-flash-lite": { "litellm_provider": "perplexity", "mode": "responses", + "supports_sampling_params": false, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 3e-07, @@ -69292,6 +69314,7 @@ "perplexity/google/gemini-3.6-flash": { "litellm_provider": "perplexity", "mode": "responses", + "supports_sampling_params": false, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 1.5e-06, @@ -69302,6 +69325,7 @@ "perplexity/google/gemini-3.7-flash": { "litellm_provider": "perplexity", "mode": "responses", + "supports_sampling_params": false, "supports_web_search": true, "supports_function_calling": true, "input_cost_per_token": 7.5e-07, @@ -71393,6 +71417,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_vision": true, "supports_web_search": true }, @@ -71410,6 +71435,7 @@ "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, + "supports_sampling_params": false, "supports_vision": true, "supports_web_search": true }, @@ -74340,6 +74366,7 @@ "perplexity/google/gemini-3.8-flash": { "litellm_provider": "perplexity", "mode": "responses", + "supports_sampling_params": false, "input_cost_per_token": 7.5e-07, "output_cost_per_token": 3.75e-06, "cache_read_input_token_cost": 7.5e-08, diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index dc1e762e752..cf4a7c4cbc8 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -12,8 +12,10 @@ from urllib.parse import parse_qs, urlparse import httpx import pytest +from pydantic import TypeAdapter from unittest.mock import MagicMock +import litellm from litellm.llms.vertex_ai.files.transformation import ( VertexAIFilesConfig, _get_litellm_batch_custom_id_from_labels, @@ -1988,3 +1990,58 @@ class TestVertexEmbeddingsBatchOutputTranslation: content = json.dumps(legacy_row).encode("utf-8") assert config._try_transform_vertex_batch_output_to_openai(content) == content + + +_SAMPLING_KNOBS: Final = MappingProxyType({"temperature": 0, "top_p": 0.5, "top_k": 3}) + + +def _chat_entry(model: str, knobs: Mapping[str, object]) -> Mapping[str, object]: + return MappingProxyType( + { + "custom_id": "request-1", + "method": "POST", + "url": "/v1/chat/completions", + "body": {"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 5, **knobs}, + } + ) + + +def _generation_config(row: Mapping[str, object]) -> object: + request: Final = TypeAdapter(dict[str, object]).validate_python(row["request"]) + return request["generationConfig"] + + +def _flag_model_as_no_sampling(monkeypatch: pytest.MonkeyPatch, model: str) -> None: + monkeypatch.setitem(litellm.model_cost, model, {"supports_sampling_params": False}) + + +def test_batch_row_for_a_flagged_gemini_model_raises_a_400_naming_drop_params(monkeypatch: pytest.MonkeyPatch) -> None: + """A batch file row gets the same contract as a real-time call: the knobs Google ignores are refused + with a 400 that names drop_params instead of silently vanishing from the uploaded row""" + _flag_model_as_no_sampling(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + with pytest.raises(litellm.UnsupportedParamsError, match=r"temperature, top_p, top_k.*drop_params"): + _wrap_entries([_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)]) + + +def test_batch_row_with_null_knobs_for_a_flagged_gemini_model_is_not_refused(monkeypatch: pytest.MonkeyPatch) -> None: + """A null knob means no override, the same as on a real-time call, so it must not fail the upload""" + _flag_model_as_no_sampling(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", False) + (row,) = _wrap_entries([_chat_entry("gemini-zeta-9", {"temperature": None, "top_p": None, "top_k": None})]) + assert _generation_config(row) == {"max_output_tokens": 5} + + +def test_batch_row_for_a_flagged_gemini_model_drops_the_knobs_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, +) -> None: + _flag_model_as_no_sampling(monkeypatch, "gemini-zeta-9") + monkeypatch.setattr(litellm, "drop_params", True) + (row,) = _wrap_entries([_chat_entry("gemini-zeta-9", _SAMPLING_KNOBS)]) + assert _generation_config(row) == {"max_output_tokens": 5} + + +def test_batch_row_for_an_unflagged_gemini_model_keeps_the_knobs(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + (row,) = _wrap_entries([_chat_entry("gemini-2.5-flash", _SAMPLING_KNOBS)]) + assert _generation_config(row) == {"max_output_tokens": 5, **_SAMPLING_KNOBS} diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 0796e125817..bb624add673 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -27,7 +27,7 @@ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, _log_gemini_sampling_params_deprecated, ) -from litellm.types.llms.vertex_ai import GeminiFinishReason, UsageMetadata +from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage from litellm.utils import _invalidate_model_cost_lowercase_map, CustomStreamWrapper from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER @@ -3739,6 +3739,169 @@ def test_get_optional_params_gemini_3_has_no_default_temperature( assert optional_params["thinkingConfig"]["thinkingLevel"] == "medium" +_SAMPLING_PARAMS: Final = {"temperature": 0.3, "top_p": 0.9, "top_k": 20} + + +def _gemini_config(config_cls: Literal["vertex", "studio"]) -> VertexGeminiConfig: + return VertexGeminiConfig() if config_cls == "vertex" else GoogleAIStudioGeminiConfig() + + +def _gemini_request_body( + custom_llm_provider: Literal["vertex_ai", "gemini"], + optional_params: dict[str, object], + litellm_params: dict[str, object], +) -> RequestBody: + from litellm.llms.vertex_ai.gemini.transformation import transform_request_body + + return transform_request_body( + messages=[{"role": "user", "content": "hi"}], + model="gemini-zeta-9", + optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + cached_content=None, + ) + + +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +@pytest.mark.parametrize("param", ["temperature", "top_p"]) +def test_get_optional_params_raises_a_400_naming_drop_params_for_a_flagged_gemini_model( + custom_llm_provider: Literal["vertex_ai", "gemini"], param: str, monkeypatch: pytest.MonkeyPatch +) -> None: + """Google ignores sampling params on models flagged ``supports_sampling_params: false``, so without + drop_params the caller gets the 400 that names the escape hatch instead of a 200 that lied (#47015)""" + from litellm.utils import get_optional_params + + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setitem(litellm.model_cost, "gemini-zeta-9", {"supports_sampling_params": False}) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params") as excinfo: + get_optional_params( + model="gemini-zeta-9", custom_llm_provider=custom_llm_provider, **{param: _SAMPLING_PARAMS[param]} + ) + + assert excinfo.value.status_code == 400 + assert param in str(excinfo.value) + + +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +def test_allowed_openai_params_still_forwards_a_sampling_param_the_map_flags_off( + custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch +) -> None: + """The 400 tells the caller to send allowed_openai_params, so that escape hatch has to work""" + from litellm.utils import get_optional_params + + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setitem(litellm.model_cost, "gemini-zeta-9", {"supports_sampling_params": False}) + + optional_params: Final = get_optional_params( + model="gemini-zeta-9", + custom_llm_provider=custom_llm_provider, + temperature=0.3, + allowed_openai_params=["temperature"], + ) + + assert optional_params["temperature"] == 0.3 + + +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +def test_top_k_is_stripped_from_the_request_body_of_a_flagged_gemini_model_under_drop_params( + custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch +) -> None: + """top_k is a provider-specific kwarg that skips the shared unsupported-param check, so it is gated + where the request body is built""" + monkeypatch.setitem(litellm.model_cost, "gemini-zeta-9", {"supports_sampling_params": False}) + + body: Final = _gemini_request_body( + custom_llm_provider, {"top_k": 20, "max_output_tokens": 10}, {"drop_params": True} + ) + + assert body["generationConfig"] == {"max_output_tokens": 10} + + +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +def test_top_k_raises_a_400_naming_drop_params_for_a_flagged_gemini_model( + custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setitem(litellm.model_cost, "gemini-zeta-9", {"supports_sampling_params": False}) + + with pytest.raises(litellm.utils.UnsupportedParamsError, match="drop_params") as excinfo: + _gemini_request_body(custom_llm_provider, {"top_k": 20}, {}) + + assert excinfo.value.status_code == 400 + assert "top_k" in str(excinfo.value) + + +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +@pytest.mark.parametrize("row", [{"supports_sampling_params": True}, {}], ids=["flag_true", "flag_absent"]) +def test_gemini_keeps_forwarding_sampling_params_unless_the_model_map_flags_them_off( + custom_llm_provider: Literal["vertex_ai", "gemini"], row: dict[str, bool], monkeypatch: pytest.MonkeyPatch +) -> None: + """An explicit true and a model the map says nothing about both keep today's pass-through, top_k included""" + from litellm.utils import get_optional_params + + monkeypatch.setitem(litellm.model_cost, "gemini-zeta-9", row) + + optional_params: Final = get_optional_params( + model="gemini-zeta-9", custom_llm_provider=custom_llm_provider, drop_params=True, **_SAMPLING_PARAMS + ) + body: Final = _gemini_request_body(custom_llm_provider, dict(optional_params), {"drop_params": True}) + + assert body["generationConfig"] == _SAMPLING_PARAMS + + +@pytest.mark.parametrize("model_key", ["gemini-zeta-9", "vertex_ai/gemini-zeta-9", "gemini/gemini-zeta-9"]) +def test_gemini_sampling_flag_is_read_from_whichever_provider_row_carries_it( + model_key: str, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(litellm.model_cost, model_key, {"supports_sampling_params": False}) + + result: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"temperature": 0.3}, + optional_params={}, + model="gemini-zeta-9", + drop_params=True, + ) + + assert "temperature" not in result + + +@pytest.mark.parametrize("config_cls", ["vertex", "studio"]) +def test_gemini_supported_openai_params_omit_sampling_params_when_flagged_off( + config_cls: Literal["vertex", "studio"], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setitem(litellm.model_cost, "gemini-zeta-9", {"supports_sampling_params": False}) + + supported: Final = set(_gemini_config(config_cls).get_supported_openai_params("gemini-zeta-9")) + + assert not supported & set(_SAMPLING_PARAMS) + assert "max_tokens" in supported + + +@pytest.mark.parametrize("custom_llm_provider", ["vertex_ai", "gemini"]) +def test_get_optional_params_drops_sampling_params_for_a_flagged_gemini_model_under_drop_params( + custom_llm_provider: Literal["vertex_ai", "gemini"], monkeypatch: pytest.MonkeyPatch +) -> None: + """The path every SDK and proxy call takes, on both the Vertex and the AI Studio route""" + from litellm.utils import get_optional_params + + monkeypatch.setitem(litellm.model_cost, "gemini-zeta-9", {"supports_sampling_params": False}) + + optional_params: Final = get_optional_params( + model="gemini-zeta-9", + custom_llm_provider=custom_llm_provider, + temperature=0.3, + top_p=0.9, + max_tokens=10, + drop_params=True, + ) + + assert "temperature" not in optional_params + assert "top_p" not in optional_params + assert optional_params["max_output_tokens"] == 10 + + def test_media_resolution_from_detail_parameter(): """Test that OpenAI's detail parameter is correctly mapped to media_resolution""" from litellm.llms.vertex_ai.gemini.transformation import ( diff --git a/tests/unit/test_model_prices_and_context_window.py b/tests/unit/test_model_prices_and_context_window.py index 628870259a4..701b79d0ba9 100644 --- a/tests/unit/test_model_prices_and_context_window.py +++ b/tests/unit/test_model_prices_and_context_window.py @@ -272,6 +272,27 @@ def test_vertex_gemini_flash_rows_carry_the_documented_retirement_date(prices: d assert prices[f"{prefix}{model}"]["deprecation_date"] == VERTEX_GEMINI_FLASH_RETIREMENT_DATES[model] +def _sampling_flags_across_provider_twins(prices: Mapping[str, Mapping[str, object]], key: str) -> frozenset[object]: + bare: Final = key.split("/")[-1] + twins: Final = tuple(twin for twin in (bare, f"vertex_ai/{bare}", f"gemini/{bare}") if twin in prices) + return frozenset(prices[twin].get("supports_sampling_params") for twin in (key, *twins)) + + +def test_gemini_rows_agree_on_supports_sampling_params_across_their_provider_twins( + prices: Mapping[str, Mapping[str, object]], +) -> None: + """The bare, vertex_ai/ and gemini/ rows of one Gemini model gate sampling params identically, so a request + takes the same drop-or-400 path on the Vertex route and the AI Studio route""" + flagged: Final = tuple( + key + for key, row in prices.items() + if "supports_sampling_params" in row and key.split("/")[-1].startswith("gemini-") + ) + assert flagged + for key in flagged: + assert len(_sampling_flags_across_provider_twins(prices, key)) == 1, key + + OPENAI_REASONING_FAMILY_MARKERS = ("codex", "deep-research", "chat-latest") diff --git a/tests/unit/test_router_model_cost_isolation.py b/tests/unit/test_router_model_cost_isolation.py index 426481d2719..f001008ee2c 100644 --- a/tests/unit/test_router_model_cost_isolation.py +++ b/tests/unit/test_router_model_cost_isolation.py @@ -585,7 +585,7 @@ def test_should_not_pollute_shared_key_with_custom_nonzero_pricing(): ) -@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill"]) +@pytest.mark.parametrize("flag", ["supports_regex_lookaround", "supports_assistant_prefill", "supports_sampling_params"]) def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None: """A deployment's capability override must not land on the shared ``{provider}/{model}`` key, or every sibling deployment of that model would inherit it.""" From 49b5444ef946277e85bfa3e4c71f4ba098ef6768 Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Tue, 20 Oct 2026 16:53:35 -0700 Subject: [PATCH 026/235] fix(mcp): list per-user auth MCP tools without a shared token (#47677) --- .../mcp_server/mcp_server_manager.py | 3 + .../outbound_credentials/adapter.py | 12 +++- .../_experimental/mcp_server/upstream.py | 3 + .../mcp_server/test_mcp_server_manager.py | 61 ++++++++++++++++++- 4 files changed, 76 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 0286096e405..7eddf592262 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -4361,6 +4361,7 @@ class MCPServerManager: raw_headers: Mapping[str, str] | None = None, client_ip: str | None = None, protocol_version_override: MCPUpstreamProtocol | None = None, + for_tool_listing: bool = False, ) -> MCPClient: self.catalog.assert_current(server) record_auth_resolution(server.server_id, AuthResolution.unresolved) @@ -4394,6 +4395,7 @@ class MCPServerManager: else None ), upstream_transport=self._upstream_transport, + for_tool_listing=for_tool_listing, ) _create_mcp_client = create_mcp_client @@ -4540,6 +4542,7 @@ class MCPServerManager: user_api_key_auth=user_api_key_auth, raw_headers=raw_headers, client_ip=client_ip, + for_tool_listing=True, ) ## HANDLE OPENAPI TOOLS diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py index f217072bd39..37404ceece8 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/adapter.py @@ -460,6 +460,8 @@ def validate_static_credential( headers: Mapping[str, str], upstream_token_header: str | None = None, static_header_names: Iterable[str] = (), + *, + allow_absent: bool = False, ) -> Result[None, CredError]: if auth_type not in _STATIC_MODES: return Ok(None) @@ -475,17 +477,23 @@ def validate_static_credential( ) ) values: Final = tuple((name.lower(), value.strip()) for name, value in headers.items() if name.lower() in slots) + if allow_absent and not values: + return Ok(None) if any(_usable_credential_value(auth_type, name, value) for name, value in values): return Ok(None) return Error(CredError.of_misconfigured(f"{auth_type} requires a usable upstream credential")) -async def prepare_mcp_client(server: MCPServer, client: MCPClient) -> MCPClient: +async def prepare_mcp_client(server: MCPServer, client: MCPClient, *, for_tool_listing: bool = False) -> MCPClient: if server.auth_type not in _STATIC_MODES or client.transport_type == MCPTransport.stdio: return client request: Final = await client.prepare_request_auth() match validate_static_credential( - server.auth_type, request.headers, server.upstream_token_header, server.static_headers or () + server.auth_type, + request.headers, + server.upstream_token_header, + server.static_headers or (), + allow_absent=for_tool_listing and server.auth_type == MCPAuth.authorization, ): case Error(error): raise_public(error) diff --git a/litellm/proxy/_experimental/mcp_server/upstream.py b/litellm/proxy/_experimental/mcp_server/upstream.py index 3b7cf904b6a..1e39c068806 100644 --- a/litellm/proxy/_experimental/mcp_server/upstream.py +++ b/litellm/proxy/_experimental/mcp_server/upstream.py @@ -208,6 +208,7 @@ async def prepare_upstream_client( sampling_callback: SamplingCallback | None = None, elicitation_callback: ElicitationCallback | None = None, upstream_transport: httpx2.AsyncBaseTransport | None = None, + for_tool_listing: bool = False, ) -> MCPClient: transport: Final[MCPTransportType] = server.transport or MCPTransport.sse server_spec: Final = None if transport == MCPTransport.stdio else to_server_spec_fail_closed(server) @@ -272,6 +273,7 @@ async def prepare_upstream_client( sampling_callback=sampling_callback, elicitation_callback=elicitation_callback, ), + for_tool_listing=for_tool_listing, ) aws_auth: Final = ( MCPSigV4Auth( @@ -316,4 +318,5 @@ async def prepare_upstream_client( sampling_callback=sampling_callback, elicitation_callback=elicitation_callback, ), + for_tool_listing=for_tool_listing, ) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index be70f595fee..7315315a619 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -16446,6 +16446,7 @@ async def test_health_check_logs_upstream_exchange_only_for_http_failures( class _DiscoveryUpstream: def __init__(self) -> None: self.requests: tuple[tuple[str, str], ...] = () + self.tools: tuple[dict[str, object], ...] = () self.outcome = "supported" self.ttl_ms = 60000 self.entered = asyncio.Event() @@ -16507,7 +16508,7 @@ class _DiscoveryUpstream: "resources/templates/list": { "resourceTemplates": [{"name": "example", "uriTemplate": "test://{name}", "description": "original"}] }, - "tools/list": {"tools": []}, + "tools/list": {"tools": list(self.tools)}, }[payload.method] continuation: Final = ( {"nextCursor": "last-page"} @@ -16544,6 +16545,64 @@ def _discovery_server() -> MCPServer: ) +@pytest.mark.asyncio +async def test_per_user_authorization_server_lists_without_static_credential() -> None: + upstream: Final = _DiscoveryUpstream() + upstream.tools = ({"name": "search", "inputSchema": {"type": "object"}},) + server: Final = MCPServer( + server_id="per-user-auth", + name="per-user-auth", + url="https://discovery.example/mcp", + transport=MCPTransport.http, + protocol_version="2026-07-28", + auth_type=MCPAuth.authorization, + ) + manager: Final = MCPServerManager() + manager.registry[server.server_id] = server + + with _mcp_upstream(upstream.respond): + tools: Final = await manager.get_tools_from_server(server, add_prefix=False) + + assert [tool.name for tool in tools] == ["search"] + assert ("tools/list", "") in upstream.requests + with pytest.raises(HTTPException, match="requires a usable upstream credential"): + await manager.create_mcp_client(server) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("auth_type", "credential", "upstream_token_header"), + ( + (MCPAuth.authorization, "Bearer", None), + (MCPAuth.authorization, "Bearer", "X-Token"), + (MCPAuth.authorization, " ", None), + (MCPAuth.authorization, " ", "X-Token"), + (MCPAuth.bearer_token, None, None), + ), +) +async def test_tool_listing_rejects_malformed_or_other_missing_static_credentials( + auth_type: MCPAuth, credential: str | None, upstream_token_header: str | None +) -> None: + upstream: Final = _DiscoveryUpstream() + server: Final = MCPServer( + server_id="invalid-static-auth", + name="invalid-static-auth", + url="https://discovery.example/mcp", + transport=MCPTransport.http, + protocol_version="2026-07-28", + auth_type=auth_type, + authentication_token=credential, + upstream_token_header=upstream_token_header, + ) + manager: Final = MCPServerManager() + manager.registry[server.server_id] = server + + with _mcp_upstream(upstream.respond), pytest.raises(MCPServerListError): + await manager.get_tools_from_server(server) + + assert upstream.requests == () + + @pytest.mark.asyncio @pytest.mark.parametrize("kind", ("prompts", "resources", "templates")) async def test_discovery_cache_reuses_raw_results_and_expires(kind: str) -> None: From 6083cf97769aeedb2719c5b5c901ded3106cc79e Mon Sep 17 00:00:00 2001 From: Yassin Kortam Date: Tue, 20 Oct 2026 16:55:02 -0700 Subject: [PATCH 027/235] fix(mcp): list per-user auth MCP tools without a shared token (#47677) From 735ebc466c95bf725e04328e613f9f845d016ffe Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:09:17 +0000 Subject: [PATCH 028/235] chore(deps-dev): bump vite from 8.3.0 to 8.3.4 in /vscode-extension (#47627) * chore(deps-dev): bump vite from 8.3.0 to 8.3.4 in /vscode-extension Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 8.3.0 to 8.3.4. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v8.3.4/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-version: 8.3.4 dependency-type: indirect ... Signed-off-by: dependabot[bot] * chore(deps): hold vscode-extension postcss at 8.5.30, outside the 3-day release cooldown --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> --- vscode-extension/package-lock.json | 160 ++++++++++++++--------------- 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/vscode-extension/package-lock.json b/vscode-extension/package-lock.json index 497c5807d5f..536d7a88f25 100644 --- a/vscode-extension/package-lock.json +++ b/vscode-extension/package-lock.json @@ -899,9 +899,9 @@ } }, "node_modules/@oxc-project/types": { - "version": "0.150.0", - "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.150.0.tgz", - "integrity": "sha512-rDS5/31E9HfPl/CIzGrn0DOlvBbXFseQ5URJ9sYMfstbKLD/c6Gm9vmRzRGDdAXyOIL4zmO37lc9RIwYqVruZw==", + "version": "0.154.0", + "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.154.0.tgz", + "integrity": "sha512-KYOUGQFlLWR+s4QMV2Q9ko38TZm7EdO87Qm7rvZLqk/webse/cDjtyxKhg/47l0byCbs/k46LT8qc4PKjAikzA==", "dev": true, "license": "MIT", "funding": { @@ -909,9 +909,9 @@ } }, "node_modules/@rolldown/binding-android-arm-eabi": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.9.tgz", - "integrity": "sha512-tNISae1QEf/vkb3xkRcjV5SEdzPE97We5IVaa2Z8jSszQPZ8U60B/YCYpw4QI7VidYsBtKavczXf+DyDs9WGxw==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm-eabi/-/binding-android-arm-eabi-1.2.14.tgz", + "integrity": "sha512-aWR+XKH9+rW7m5fWq/5360J3mHycrx/o6xZM2DQFdJnTMdyP7/WzxPjhtLbVWdKo2y612/4U1bWqb3HQhiF7vg==", "cpu": [ "arm" ], @@ -926,9 +926,9 @@ } }, "node_modules/@rolldown/binding-android-arm64": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.9.tgz", - "integrity": "sha512-YC8YsI30o606GTZi0VyzYlsDKFP8W61i/QzayHDkLbNEz/IShqAmTa+hsJRj13xTHA0H+6fk4b2UmGn+Q/cMlg==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-android-arm64/-/binding-android-arm64-1.2.14.tgz", + "integrity": "sha512-QiVxHofl6A8lQU7WAtOUY4QMVZXohxwz7WJJoeh8NBEUyKfIkMRKhYyq8az/1RypNiIXzNTtiS6KwHz0erxYRQ==", "cpu": [ "arm64" ], @@ -943,9 +943,9 @@ } }, "node_modules/@rolldown/binding-darwin-arm64": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.9.tgz", - "integrity": "sha512-IwhlH3qK5urrY8hZiEgGkHKEFN901p/p2bjxCxJlr4GyNnF7wYpUvK+Y43uaRYuC4hpfjzbR3SJC3arX1jGvmw==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-arm64/-/binding-darwin-arm64-1.2.14.tgz", + "integrity": "sha512-lTQOzHuBIooh3jIr5XINQkjPGEuGy+Dsy0meXDr6Sjty0b99kYgGh4uBC/8mJX1T0kC8d6S0aPP1Y+bHkh6aBw==", "cpu": [ "arm64" ], @@ -960,9 +960,9 @@ } }, "node_modules/@rolldown/binding-darwin-x64": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.9.tgz", - "integrity": "sha512-XxpJfVzFh+jilRxIXUqcfYAYcunIc/XEzIizsOL1fcJee5Sf7H3mH8WlLmfHfluz5amqR88QQo9izKtmMlavAw==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-darwin-x64/-/binding-darwin-x64-1.2.14.tgz", + "integrity": "sha512-hyMKEGatjmBYCs/Q4sbIcL/GqpFEqsX3wc+HYVQ/85A4rxc1o7rPm4DUI4GV4k/h0ITddwFSGpqQunI14GXjTA==", "cpu": [ "x64" ], @@ -977,9 +977,9 @@ } }, "node_modules/@rolldown/binding-freebsd-x64": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.9.tgz", - "integrity": "sha512-kSfvhmgeWyfkbT3p/1s5vSgboogoah2zkm9fX2zjg2hHxSV7T4KhMWRUUaRk4OXNqoD3QAUeRqLcs1aZOK4U1g==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-freebsd-x64/-/binding-freebsd-x64-1.2.14.tgz", + "integrity": "sha512-u371GbXoVXrEgP20PwzoGNOdAOqAMhzo19JzzY+V4YmmnC+DTOoqdbMu+O+54emM59Mik/UdrtWgcuEDKWRPKA==", "cpu": [ "x64" ], @@ -994,9 +994,9 @@ } }, "node_modules/@rolldown/binding-linux-arm-gnueabihf": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.9.tgz", - "integrity": "sha512-1RVzG17pxqbTfYLC352JlLt6kKLG+6Hr30n8DlIJqsnV5luUDd2Qdx9Ayw1Cabfyb1K9k0jXEZ7evxkRoT+uiw==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm-gnueabihf/-/binding-linux-arm-gnueabihf-1.2.14.tgz", + "integrity": "sha512-BqwKsdZQf6EqVWr8fv2MrDYzAvOZ7+NYFSJESRGs87fkseEhs0TU9TXgRJTkZoRRkxTizyEg1LKUXfU3/78uqg==", "cpu": [ "arm" ], @@ -1011,9 +1011,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-gnu": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.9.tgz", - "integrity": "sha512-BXqPvZ2drqVD+/Z8UpKwcs4Mp7grM+eGFku4CAEKrEtcbAsUpzREphK1sogCRZGreVPiMkiiBtw0n3TPteuqvw==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-gnu/-/binding-linux-arm64-gnu-1.2.14.tgz", + "integrity": "sha512-SD9kY8M009x/7s57Z1dgUVlP5+uYBgKQK0Tkr1WIWngrtUJmRCW1RJFMY7QDla2khXhmg2i5Io7MUjcmZNZ/KQ==", "cpu": [ "arm64" ], @@ -1031,9 +1031,9 @@ } }, "node_modules/@rolldown/binding-linux-arm64-musl": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.9.tgz", - "integrity": "sha512-11vWvo8YDwLzukt27J3aYDWU+gg2P7J+ZOmiJ0hkF5BXZDW7pVya7r40MXDy6ya0i9KamoENSVKIugvJNgFXIA==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-arm64-musl/-/binding-linux-arm64-musl-1.2.14.tgz", + "integrity": "sha512-BJ2CL19M6uuWesMgnqJzgxoI+N+eV7b0A9O9vLV42y+gitC41fNa8KElau1tHVzuLWX5GzEuxMPhAO0tQXQMkg==", "cpu": [ "arm64" ], @@ -1051,9 +1051,9 @@ } }, "node_modules/@rolldown/binding-linux-ppc64-gnu": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.9.tgz", - "integrity": "sha512-a1tijMkdwsIARtc0F39ApURROkf3NwqinI6TOiSSWCTR7dT96dffNvMUtDHnq64wKNTIZOIlzKrFvvFUznJiyw==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-ppc64-gnu/-/binding-linux-ppc64-gnu-1.2.14.tgz", + "integrity": "sha512-4+Kl8daPV8NDHaD7mfCcc1pRM+Qg7ab8DyMm5FG5WoZgv4h2i3KFN1CybYBNbQqmuYm7ftJLvmMI1eHYJ22S/Q==", "cpu": [ "ppc64" ], @@ -1071,9 +1071,9 @@ } }, "node_modules/@rolldown/binding-linux-s390x-gnu": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.9.tgz", - "integrity": "sha512-x6SQNdAvv4c3hWqTMaWuawzMX9myaCs/yEmlGsxJzkdClnHW7FbrjQuSiRDhuSYzEYoEMhsaJy9qHG/XNemJPQ==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-s390x-gnu/-/binding-linux-s390x-gnu-1.2.14.tgz", + "integrity": "sha512-V/h8ePq1fKDrmtFMz6xcCN3DKvuvMRrwwD1q0PkBeeo4DpK64biaKvqWGVxqCeYKqeRk7gE7/iKPxLCn08O97A==", "cpu": [ "s390x" ], @@ -1091,9 +1091,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-gnu": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.9.tgz", - "integrity": "sha512-9s0AZ8BFK5/n7B/TBoa2yJE3gI3KURrbXcPBlsAsvjU4VeJKgE90y1YtNxyEUIcHPQkg6/yfF3qihUrcM/Kf0Q==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-gnu/-/binding-linux-x64-gnu-1.2.14.tgz", + "integrity": "sha512-WxalXzgrLVpKC93xfpaJ3gCXUWzeCr+FRCbDVD2mUAy4L6Vo3lNVz0epUEUpxPWJkLNYUwM4OSIikv36OJ9mLg==", "cpu": [ "x64" ], @@ -1111,9 +1111,9 @@ } }, "node_modules/@rolldown/binding-linux-x64-musl": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.9.tgz", - "integrity": "sha512-P7VWAmV+WdJluH7ovnRGoiv2i8To7GAZ+kGzfGup635cyL7SyYl3lSUaA3Gp5THf0n/Co5EyEqb2zbqq+nMOHQ==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-linux-x64-musl/-/binding-linux-x64-musl-1.2.14.tgz", + "integrity": "sha512-3G3tnAmTpdk9jO3GQtyHXqeFD2SlecfTws/Y3M2mmj9nMBwwuLT4kur8sZKeu7ENUHSpVO3cIsm5A52LEWWwlg==", "cpu": [ "x64" ], @@ -1131,9 +1131,9 @@ } }, "node_modules/@rolldown/binding-openharmony-arm64": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.9.tgz", - "integrity": "sha512-1qixtsE4BK8h+yS3BfmZ09UhA7O/N4IACva6YBr7EBvCJraByTuRcgOTaiA62Tm0vey3UcKXLOaoGHtYmNGEVg==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-openharmony-arm64/-/binding-openharmony-arm64-1.2.14.tgz", + "integrity": "sha512-qyr5XiNGxm2ytx17vvtcJ9nXn6v1gENETi503bJx3OGwZ1tSonj7BosxbMa/xeAfSl5iS9X6KjEWYXLu6v1L7Q==", "cpu": [ "arm64" ], @@ -1148,9 +1148,9 @@ } }, "node_modules/@rolldown/binding-win32-arm64-msvc": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.9.tgz", - "integrity": "sha512-ok8IQjcEPs1AKZfuEUznVBrJw+gK4soq+bx8b1X2XoMqVClarc1q5JDmVtWXY1xfr6ZuHTAsPXHTgTrqKTZeww==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-arm64-msvc/-/binding-win32-arm64-msvc-1.2.14.tgz", + "integrity": "sha512-oA4o5o0S2yB199YyYXDzKUv1Z56Rc+mgOjwm7Z/vJ9Z8igiDPYvoOuDyo6g7AK/ZaF09nPhDKVnGBcYHO11GXA==", "cpu": [ "arm64" ], @@ -1165,9 +1165,9 @@ } }, "node_modules/@rolldown/binding-win32-x64-msvc": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.9.tgz", - "integrity": "sha512-Ip2mXoU0hM0boq3Rf+ekuT653OROSo6aSYcPT1VHE4q52KvyxgFkQgrgb/IEsxOuvQ2fZZbs8khJAyCEPM24/g==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/@rolldown/binding-win32-x64-msvc/-/binding-win32-x64-msvc-1.2.14.tgz", + "integrity": "sha512-9mE94bf2GZUaR/fUSXCMpBoWkctu421Mi0ORpm4zqfbkjKep7RxzPFa5etcGs5h+2KApoWBOA/hAJa/7saGgEQ==", "cpu": [ "x64" ], @@ -2730,9 +2730,9 @@ "license": "ISC" }, "node_modules/nanoid": { - "version": "3.3.19", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.19.tgz", - "integrity": "sha512-Y2tUNy4ouw6tq5oDSKeQYGOyhkUBhNOcGV/02KC+6kd9eDGqdZd++mjMiIDilrBYvjEnCYvVtsuHCuP+okSfug==", + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", "dev": true, "funding": [ { @@ -2879,9 +2879,9 @@ } }, "node_modules/postcss": { - "version": "8.5.28", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.28.tgz", - "integrity": "sha512-RRuzqDtt5Y9h3quz5hWhK+TPnsmVs6WwSU6LkJMeY4HstUEDuYTG8UJSdawMRzmzAtV+KEoG8N3Qg2qLy5vM/A==", + "version": "8.5.30", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.30.tgz", + "integrity": "sha512-zkd+e9S1AQHhmaBKye9zlFHWjOwVgqXl1q3ECYy8561K7qxR2cREf/STLskulBFd78k6AOFeA+ff1TtEzk9bAA==", "dev": true, "funding": [ { @@ -2899,9 +2899,9 @@ ], "license": "MIT", "dependencies": { - "nanoid": "^3.3.18", + "nanoid": "^3.3.20", "picocolors": "^1.1.1", - "source-map-js": "^1.2.1" + "source-map-js": "^1.2.2" }, "engines": { "node": "^10 || ^12 || >=14" @@ -2960,13 +2960,13 @@ } }, "node_modules/rolldown": { - "version": "1.2.9", - "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.9.tgz", - "integrity": "sha512-hx/Pv0N1haXRb11qkfnK5MXB/iqr7i0yjWQqmO9uHqZpBgQSqzc8UsSnEpalsh+j1I8qQ2CkXAkJC8Br3dKSlg==", + "version": "1.2.14", + "resolved": "https://registry.npmjs.org/rolldown/-/rolldown-1.2.14.tgz", + "integrity": "sha512-eTBTTqMR6uHKLgcUqpJP8ntzDI6V4LYA8negAMBEI7gkgVvOdrx6VsvS2Ym+83KhOPdLCHWJu0w7UQFJZhHKNw==", "dev": true, "license": "MIT", "dependencies": { - "@oxc-project/types": "=0.150.0", + "@oxc-project/types": "=0.154.0", "@rolldown/pluginutils": "^1.0.0" }, "bin": { @@ -2976,21 +2976,21 @@ "node": "^20.19.0 || >=22.12.0" }, "optionalDependencies": { - "@rolldown/binding-android-arm-eabi": "1.2.9", - "@rolldown/binding-android-arm64": "1.2.9", - "@rolldown/binding-darwin-arm64": "1.2.9", - "@rolldown/binding-darwin-x64": "1.2.9", - "@rolldown/binding-freebsd-x64": "1.2.9", - "@rolldown/binding-linux-arm-gnueabihf": "1.2.9", - "@rolldown/binding-linux-arm64-gnu": "1.2.9", - "@rolldown/binding-linux-arm64-musl": "1.2.9", - "@rolldown/binding-linux-ppc64-gnu": "1.2.9", - "@rolldown/binding-linux-s390x-gnu": "1.2.9", - "@rolldown/binding-linux-x64-gnu": "1.2.9", - "@rolldown/binding-linux-x64-musl": "1.2.9", - "@rolldown/binding-openharmony-arm64": "1.2.9", - "@rolldown/binding-win32-arm64-msvc": "1.2.9", - "@rolldown/binding-win32-x64-msvc": "1.2.9" + "@rolldown/binding-android-arm-eabi": "1.2.14", + "@rolldown/binding-android-arm64": "1.2.14", + "@rolldown/binding-darwin-arm64": "1.2.14", + "@rolldown/binding-darwin-x64": "1.2.14", + "@rolldown/binding-freebsd-x64": "1.2.14", + "@rolldown/binding-linux-arm-gnueabihf": "1.2.14", + "@rolldown/binding-linux-arm64-gnu": "1.2.14", + "@rolldown/binding-linux-arm64-musl": "1.2.14", + "@rolldown/binding-linux-ppc64-gnu": "1.2.14", + "@rolldown/binding-linux-s390x-gnu": "1.2.14", + "@rolldown/binding-linux-x64-gnu": "1.2.14", + "@rolldown/binding-linux-x64-musl": "1.2.14", + "@rolldown/binding-openharmony-arm64": "1.2.14", + "@rolldown/binding-win32-arm64-msvc": "1.2.14", + "@rolldown/binding-win32-x64-msvc": "1.2.14" } }, "node_modules/run-applescript": { @@ -3312,16 +3312,16 @@ } }, "node_modules/vite": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.0.tgz", - "integrity": "sha512-lhZBVvEHefgE+HQZC9O7EBJgCU/nVzFNl7vkS4RE0APtWLP02/8QVIkQtzBxPquh7lq5/78NHipTj7ODQ6XuyQ==", + "version": "8.3.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-8.3.4.tgz", + "integrity": "sha512-IftLE8phJhdNRIGNEV4JCCvfNScoZQVfcXaN8TQropWGJtOHcu+iAB9pHcuGvmEb/kddw0LBe/jS8ztdC6xdiQ==", "dev": true, "license": "MIT", "dependencies": { "lightningcss": "^1.33.0", "picomatch": "^4.0.7", - "postcss": "^8.5.28", - "rolldown": "~1.2.6", + "postcss": "^8.5.29", + "rolldown": "~1.2.12", "tinyglobby": "^0.2.17" }, "bin": { From 17db64ef78aa2a3d8517422b4a2a1636c0cdd409 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 17:10:39 -0700 Subject: [PATCH 029/235] test(perf): record the row count and size of each table the Admin UI journeys query (#47683) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/setup/ui_query_fixture.py | 54 +++++++++++++++++++ tests/e2e/perf/test_ui_journeys.py | 5 +- .../perf/setup/test_ui_journeys.py | 20 ++++++- 3 files changed, 76 insertions(+), 3 deletions(-) diff --git a/tests/e2e/perf/setup/ui_query_fixture.py b/tests/e2e/perf/setup/ui_query_fixture.py index cfffa0822e8..ed941a096bd 100644 --- a/tests/e2e/perf/setup/ui_query_fixture.py +++ b/tests/e2e/perf/setup/ui_query_fixture.py @@ -3,7 +3,9 @@ from __future__ import annotations import hashlib import os import time +from collections.abc import Iterable from datetime import datetime, timedelta +from itertools import chain from typing import Final import psycopg @@ -18,6 +20,33 @@ class UiFixture(BaseModel): anchor: AwareDatetime api_key: str + """`perf_ui_db_*` JUnit properties naming how much data the journeys ran against.""" + volume: tuple[tuple[str, int], ...] = () + + +class TableVolume(BaseModel): + model_config = ConfigDict(frozen=True) + + table: str + rows: int + bytes: int + + +QUERIED_TABLES: Final = ( + "LiteLLM_SpendLogs", + "LiteLLM_DailyUserSpend", + "LiteLLM_DailyTeamSpend", + "LiteLLM_DailyTagSpend", + "LiteLLM_VerificationToken", + "LiteLLM_TeamTable", + "LiteLLM_UserTable", + "LiteLLM_ProxyModelTable", +) +_VOLUME_SQL: Final = """ + SELECT c.relname, GREATEST(c.reltuples, 0)::bigint, pg_total_relation_size(c.oid) + FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = 'public' AND c.relname = ANY(%(tables)s) +""" SEEDED_TEAMS: Final = 100 @@ -59,6 +88,30 @@ _SEED_SQL: Final = ( ) +def volume_properties(volumes: Iterable[TableVolume]) -> tuple[tuple[str, int], ...]: + """One `perf_ui_db_rows_` and one `perf_ui_db_bytes_
` property per table the journeys read, + so the dashboard can say what the latencies were measured against.""" + return tuple( + chain.from_iterable( + ((f"perf_ui_db_rows_{v.table}", v.rows), (f"perf_ui_db_bytes_{v.table}", v.bytes)) for v in volumes + ) + ) + + +def table_volumes(connection: psycopg.Connection, spend_log_rows: int) -> tuple[TableVolume, ...]: + """Planner row estimates and on-disk size (indexes included) of the queried tables. The restore's + `LiteLLM_SpendLogs` count comes from the seed checkpoint, exact where the estimate can be stale.""" + adapter: Final = TypeAdapter(tuple[str, int, int]) + found: Final = tuple( + adapter.validate_python(row) + for row in connection.execute(_VOLUME_SQL, {"tables": list(QUERIED_TABLES)}).fetchall() + ) + return tuple( + TableVolume(table=table, rows=spend_log_rows if table == "LiteLLM_SpendLogs" else rows, bytes=size) + for table, rows, size in sorted(found) + ) + + def prepare_fixture() -> UiFixture: validate_target(os.environ["DATABASE_URL"]) _finish_upgrade() @@ -71,6 +124,7 @@ def prepare_fixture() -> UiFixture: anchor=TypeAdapter(datetime).validate_python(row[0]), api_key=hashlib.md5(b"perf-key:0", usedforsecurity=False).hexdigest() + hashlib.md5(b"perf-key-tail:0", usedforsecurity=False).hexdigest(), + volume=volume_properties(table_volumes(connection, TypeAdapter(int).validate_python(row[1]))), ) return fixture diff --git a/tests/e2e/perf/test_ui_journeys.py b/tests/e2e/perf/test_ui_journeys.py index db25608a00d..63eefedec68 100644 --- a/tests/e2e/perf/test_ui_journeys.py +++ b/tests/e2e/perf/test_ui_journeys.py @@ -11,14 +11,14 @@ from types import FrameType from typing import Final import pytest -from playwright.sync_api import Page from e2e_config import MASTER_KEY, UI_BASE_URL, UI_PASSWORD, UI_USERNAME from e2e_http import send_text_external, without_retries +from playwright.sync_api import Page from setup.ui_journeys import ( - assert_within_ceiling, Dashboard, Journey, Sample, + assert_within_ceiling, dashboard_page, database_failures, error_code, @@ -111,6 +111,7 @@ def test_ui_journey_latency(journey_name: str, ui_fixture: UiFixture, request: p item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped assert isinstance(item, pytest.Item) item.user_properties.extend(ui_junit_properties(samples)) + item.user_properties.extend(ui_fixture.volume) after: Final = settled_counter(lambda: _metrics(transport), evidence) item.user_properties.extend( ( diff --git a/tests/e2e_harness/perf/setup/test_ui_journeys.py b/tests/e2e_harness/perf/setup/test_ui_journeys.py index dd2c1fc0301..d008368052c 100644 --- a/tests/e2e_harness/perf/setup/test_ui_journeys.py +++ b/tests/e2e_harness/perf/setup/test_ui_journeys.py @@ -6,9 +6,9 @@ import pytest from setup.ui_journeys import ( UI_JOURNEY_LIMITS, Sample, - error_code, assert_within_ceiling, database_failures, + error_code, journeys, summarize, ui_junit_properties, @@ -153,3 +153,21 @@ def test_failed_data_request_status_becomes_the_error_code() -> None: metrics: Final = dict(ui_junit_properties((gateway_error, slow, hung))) assert metrics["perf_ui_error_http_500"] == 1 and metrics["perf_ui_error_wall_clock"] == 1 assert "perf_ui_error_ui" not in metrics + + +def test_volume_properties_name_rows_and_bytes_per_queried_table() -> None: + from setup.ui_query_fixture import TableVolume, volume_properties + + properties: Final = volume_properties( + ( + TableVolume(table="LiteLLM_SpendLogs", rows=850_000_000, bytes=1_000_000_000_000), + TableVolume(table="LiteLLM_VerificationToken", rows=5_000, bytes=4_194_304), + ) + ) + + assert properties == ( + ("perf_ui_db_rows_LiteLLM_SpendLogs", 850_000_000), + ("perf_ui_db_bytes_LiteLLM_SpendLogs", 1_000_000_000_000), + ("perf_ui_db_rows_LiteLLM_VerificationToken", 5_000), + ("perf_ui_db_bytes_LiteLLM_VerificationToken", 4_194_304), + ) From adf337338172ce7ea6edbd80851f68b6621af2f5 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 17:36:08 -0700 Subject: [PATCH 030/235] fix(ui): show a failed state with Retry on the Logs table, Usage totals and Caching chart when their fetch errors (#47671) * fix(ui): show a failed state with Retry in DataTable when the fetch errors instead of an empty table * fix(ui): show the shared failed state with Retry on the Usage spend totals and the Caching activity chart when their fetch errors * refactor(ui): name the settled-fetch objects so the inline object arg lint budget holds * test(ui): find the caching chart card through the existing cardTitled helper --------- Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../_components/cache_dashboard.test.tsx | 21 +++++++++ .../caching/_components/cache_dashboard.tsx | 43 +++++++++++++------ .../EntityUsage/EntityUsage.test.tsx | 7 +-- .../components/EntityUsage/EntityUsage.tsx | 22 +++------- .../components/UsagePageView.test.tsx | 17 +++++++- .../_components/components/UsagePageView.tsx | 10 ++--- .../hooks/useAggregatedDailyActivity.test.ts | 13 ++++++ .../hooks/useAggregatedDailyActivity.ts | 24 +++++++++-- .../logs/request/RequestLogsPanel.test.tsx | 9 ++++ .../logs/request/RequestLogsPanel.tsx | 2 + .../logs/request/RequestLogsTable.tsx | 7 +++ .../shared/DataTable/DataTable.test.tsx | 28 ++++++++++++ .../components/shared/DataTable/DataTable.tsx | 11 +++++ .../src/components/shared/DataTable/types.ts | 2 + .../src/components/shared/ErrorState.tsx | 26 +++++++++++ 15 files changed, 200 insertions(+), 42 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/shared/ErrorState.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx index 3447602dd79..ece32449041 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.test.tsx @@ -277,4 +277,25 @@ describe("CacheDashboard cache analytics charts", () => { expect(compactTicks(requestsCard).length).toBeGreaterThan(0); expect(compactTicks(tokensCard)).toContain("60K"); }); + + it("shows the failed state with the server message and Retry when the activity query errors", async () => { + const refetch = vi.fn(); + const failedActivity = { + data: undefined, + isError: true, + error: { error: { message: "Request timed out after 30s" } }, + refetch, + }; + useCacheActivity.mockReturnValue(failedActivity); + renderDashboard(); + + await screen.findByText(REQUESTS_CHART_TITLE); + const card = cardTitled(REQUESTS_CHART_TITLE); + const alert = within(card).getByRole("alert"); + expect(alert).toHaveTextContent("Failed to load cache activity"); + expect(alert).toHaveTextContent("Request timed out after 30s"); + expect(within(card).getByText(/Click a red failed-requests segment/)).toBeInTheDocument(); + fireEvent.click(within(alert).getByRole("button", { name: "Retry" })); + expect(refetch).toHaveBeenCalledTimes(1); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx index 29819adf524..9e730be4df3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/caching/_components/cache_dashboard.tsx @@ -21,6 +21,8 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { RefreshCw } from "lucide-react"; import { cachingHealthCheckCall } from "@/components/networking"; +import { deriveErrorMessage } from "@/lib/http/client"; +import { ErrorState } from "@/components/shared/ErrorState"; import { useCacheActivity, type CacheActivityGroup } from "@/app/(dashboard)/hooks/caching/useCacheActivity"; // Import the new component @@ -94,7 +96,12 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole const [lastRefreshed, setLastRefreshed] = useState(""); const [healthCheckResponse, setHealthCheckResponse] = useState(""); - const { data: activity, refetch } = useCacheActivity({ + const { + data: activity, + isError: activityFailed, + error: activityError, + refetch, + } = useCacheActivity({ startDate: formatDateWithoutTZ(dateValue.from), endDate: formatDateWithoutTZ(dateValue.to), keyAliases: selectedApiKeys, @@ -295,19 +302,27 @@ const CacheDashboard: React.FC = ({ accessToken, token, userRole Click a red failed-requests segment to see which error codes caused those failures.

{hasUnknownGroup &&

{UNKNOWN_CALL_TYPE_NOTE}

} - { - if (item.categoryClicked === REQUEST_SERIES.failed) setErrorDrilldownCallType(item.name); - }} - /> + {activityFailed ? ( + void refetch()} + /> + ) : ( + { + if (item.categoryClicked === REQUEST_SERIES.failed) setErrorDrilldownCallType(item.name); + }} + /> + )} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx index cdee86ed559..04b4dcf9be1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.test.tsx @@ -1356,9 +1356,10 @@ describe("EntityUsage", () => { render(); - await waitFor(() => { - expect(screen.getAllByText(/Fetching spend data failed/).length).toBeGreaterThan(0); - }); + const alert = await screen.findByRole("alert"); + expect(alert).toHaveTextContent("Failed to load spend data"); + expect(alert).toHaveTextContent("aggregated unavailable"); + expect(within(alert).getByRole("button", { name: "Retry" })).toBeInTheDocument(); expect(mockDailyActivityAggregatedCall.mock.calls.filter((c) => c[0] === "team")).toHaveLength(1); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx index 8518f52e165..a75afd63633 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx @@ -14,7 +14,7 @@ import { hasCapability, type Capability } from "@/utils/capabilities"; import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; import { Bot, Boxes, ChevronDown, ChevronRight, ExternalLink, Info, KeyRound, Layers, Server } from "lucide-react"; import type { ColumnDef } from "@tanstack/react-table"; -import { Alert, AlertDescription } from "@/components/shared/Alert"; +import { ErrorState } from "@/components/shared/ErrorState"; import { ChartLoader } from "@/components/shared/chart_loader"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui/tooltip"; @@ -189,6 +189,8 @@ const EntityUsage: React.FC = ({ data: spendDataRaw, loading, failed, + errorMessage, + reload, } = useAggregatedDailyActivity({ fetch: () => api.aggregated(request as DailyActivityRequest), enabled: enabled && request !== null, @@ -208,6 +210,8 @@ const EntityUsage: React.FC = ({ data: agentSpendDataRaw, loading: agentLoading, failed: agentFailed, + errorMessage: agentErrorMessage, + reload: reloadAgent, } = useAggregatedDailyActivity({ fetch: () => ENTITY_API.agent.aggregated(agentRequest as DailyActivityRequest), enabled: enabled && showAgentBreakdown && agentRequest !== null, @@ -663,21 +667,9 @@ const EntityUsage: React.FC = ({ return (
- {failed && ( - - - Fetching spend data failed, so the totals below may be empty rather than final. Reload the page to try - again. - - - )} + {failed && } {showAgentBreakdown && agentFailed && ( - - - Fetching agent data failed, so the totals below may be empty rather than final. Reload the page to try - again. - - + )}
{ expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalled(); }); expect(mockDailyActivityAggregatedCall.mock.calls.filter((c) => c[0] === "user")).toHaveLength(1); - expect(await screen.findByText(/Fetching spend data failed/)).toBeInTheDocument(); + const alert = await screen.findByRole("alert"); + expect(alert).toHaveTextContent("Failed to load spend data"); + expect(alert).toHaveTextContent("Aggregated endpoint not available"); + expect(within(alert).getByRole("button", { name: "Retry" })).toBeInTheDocument(); + }); + + it("re-issues the aggregated call when Retry is clicked", async () => { + mockUserDailyActivityAggregatedCall.mockRejectedValue(new Error("Aggregated endpoint not available")); + + renderWithProviders(); + + const alert = await screen.findByRole("alert"); + const callsBefore = mockUserDailyActivityAggregatedCall.mock.calls.length; + fireEvent.click(within(alert).getByRole("button", { name: "Retry" })); + + await waitFor(() => expect(mockUserDailyActivityAggregatedCall.mock.calls.length).toBe(callsBefore + 1)); }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index 84cc9359ded..feb1adb0f1d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -13,6 +13,7 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from "react" import { StackedUsageChart } from "@/components/shared/charts"; import { Alert, AlertAction, AlertDescription, AlertTitle } from "@/components/shared/Alert"; +import { ErrorState } from "@/components/shared/ErrorState"; import { Button } from "@/components/ui/button"; import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; @@ -218,6 +219,8 @@ const UsagePage: React.FC = ({ teams, organizations }) => { data: aggregatedRaw, loading: aggregatedLoading, failed: aggregatedFailed, + errorMessage: aggregatedError, + reload: reloadAggregated, } = useAggregatedDailyActivity({ fetch: () => ENTITY_API.user.aggregated(dailyActivityRequest as DailyActivityRequest), enabled: dailyActivityRequest !== null, @@ -415,12 +418,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => {
{aggregatedFailed && ( - - - Fetching spend data failed, so the totals below may be empty rather than final. Reload the page to try - again. - - + )}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity.test.ts index 16a45d03db9..7e667230cb0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity.test.ts @@ -44,6 +44,19 @@ describe("useAggregatedDailyActivity", () => { await waitFor(() => expect(result.current.failed).toBe(true)); expect(result.current.loading).toBe(false); + expect(result.current.errorMessage).toBe("boom"); + }); + + it("reload fetches again after a failure and clears failed on success", async () => { + const fetch = vi.fn().mockRejectedValueOnce(new Error("boom")).mockResolvedValueOnce({ results: [], metadata: {} }); + const { result } = renderHook(() => useAggregatedDailyActivity({ fetch, enabled: true, deps: ["a"] })); + + await waitFor(() => expect(result.current.failed).toBe(true)); + act(() => result.current.reload()); + + await waitFor(() => expect(result.current.failed).toBe(false)); + expect(fetch).toHaveBeenCalledTimes(2); + expect(result.current.errorMessage).toBeNull(); }); it("ignores an out-of-order resolution from the previous deps", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity.ts index 08aec2e77f2..482041c900d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useAggregatedDailyActivity.ts @@ -14,16 +14,31 @@ interface Result { data: DailyActivityAggregatedResponse; loading: boolean; failed: boolean; + errorMessage: string | null; + reload: () => void; } interface SettledFetch { key: string; data: DailyActivityAggregatedResponse; failed: boolean; + errorMessage: string | null; } +const settledFetch = ( + key: string, + data: DailyActivityAggregatedResponse, + errorMessage: string | null, +): SettledFetch => ({ + key, + data, + failed: errorMessage !== null, + errorMessage, +}); + export function useAggregatedDailyActivity({ fetch, enabled, deps }: Options): Result { const [settled, setSettled] = useState(null); + const [attempt, setAttempt] = useState(0); const requestIdRef = useRef(0); const fetchRef = useRef(fetch); @@ -31,7 +46,7 @@ export function useAggregatedDailyActivity({ fetch, enabled, deps }: Options): R fetchRef.current = fetch; }); - const depsKey = JSON.stringify(deps); + const depsKey = JSON.stringify([attempt, ...deps]); useEffect(() => { if (!enabled) return; @@ -43,12 +58,13 @@ export function useAggregatedDailyActivity({ fetch, enabled, deps }: Options): R .current() .then((response) => { if (isStale()) return; - setSettled({ key: depsKey, data: response, failed: false }); + setSettled(settledFetch(depsKey, response, null)); }) .catch((error) => { if (isStale()) return; console.error("Error fetching daily activity:", error); - setSettled({ key: depsKey, data: EMPTY_DAILY_ACTIVITY_RESPONSE, failed: true }); + const message = error instanceof Error ? error.message : String(error); + setSettled(settledFetch(depsKey, EMPTY_DAILY_ACTIVITY_RESPONSE, message)); }); return () => { @@ -61,5 +77,7 @@ export function useAggregatedDailyActivity({ fetch, enabled, deps }: Options): R data: current?.data ?? EMPTY_DAILY_ACTIVITY_RESPONSE, loading: enabled && current === null, failed: current?.failed ?? false, + errorMessage: current?.errorMessage ?? null, + reload: () => setAttempt((n) => n + 1), }; } diff --git a/ui/litellm-dashboard/src/components/logs/request/RequestLogsPanel.test.tsx b/ui/litellm-dashboard/src/components/logs/request/RequestLogsPanel.test.tsx index d950859f22e..ce136c1cf77 100644 --- a/ui/litellm-dashboard/src/components/logs/request/RequestLogsPanel.test.tsx +++ b/ui/litellm-dashboard/src/components/logs/request/RequestLogsPanel.test.tsx @@ -803,4 +803,13 @@ describe("RequestLogsPanel", () => { expect(screen.queryByText("Refreshing every 15s")).not.toBeInTheDocument(); }); }); + + it("a failed logs request says so instead of showing an empty table", async () => { + vi.mocked(uiSpendLogsCall).mockRejectedValue(new Error("Request timed out after 30s")); + renderPanel(); + const alert = await screen.findByRole("alert", {}, { timeout: 3000 }); + expect(alert).toHaveTextContent("Failed to load"); + expect(alert).toHaveTextContent("Request timed out after 30s"); + expect(screen.getByRole("button", { name: "Retry" })).toBeEnabled(); + }); }); diff --git a/ui/litellm-dashboard/src/components/logs/request/RequestLogsPanel.tsx b/ui/litellm-dashboard/src/components/logs/request/RequestLogsPanel.tsx index 605ff373c7c..e29c48b94c0 100644 --- a/ui/litellm-dashboard/src/components/logs/request/RequestLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/logs/request/RequestLogsPanel.tsx @@ -301,6 +301,8 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, data={rows} rowCount={rowCount} isLoading={logsQuery.isLoading} + isError={logsQuery.isError} + errorMessage={logsQuery.error instanceof Error ? logsQuery.error.message : undefined} isRefreshing={logsQuery.isFetching} pagination={pagination} onPaginationChange={handlePaginationChange} diff --git a/ui/litellm-dashboard/src/components/logs/request/RequestLogsTable.tsx b/ui/litellm-dashboard/src/components/logs/request/RequestLogsTable.tsx index 822bf73302b..d3d6b43e2e3 100644 --- a/ui/litellm-dashboard/src/components/logs/request/RequestLogsTable.tsx +++ b/ui/litellm-dashboard/src/components/logs/request/RequestLogsTable.tsx @@ -18,6 +18,8 @@ interface RequestLogsTableProps { data: LogEntry[]; rowCount: number; isLoading: boolean; + isError?: boolean; + errorMessage?: ReactNode; isRefreshing: boolean; pagination: PaginationState; onPaginationChange: OnChangeFn; @@ -66,6 +68,8 @@ export function RequestLogsTable({ data, rowCount, isLoading, + isError = false, + errorMessage, isRefreshing, pagination, onPaginationChange, @@ -112,6 +116,9 @@ export function RequestLogsTable({ columnFilters={columnFilters} onColumnFiltersChange={onColumnFiltersChange} isLoading={isLoading} + isError={isError} + errorMessage={errorMessage} + onRetry={onRefresh} loadingMessage="Loading request logs…" noDataMessage={} size="compact" diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx index 05ba7e04c39..959b5ea2375 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.test.tsx @@ -254,6 +254,34 @@ describe("DataTable sorting", () => { }); }); +describe("DataTable error state", () => { + it("shows the failure and a Retry button instead of the empty state when the fetch failed", async () => { + const onRetry = vi.fn(); + render( + , + ); + const alert = screen.getByRole("alert"); + expect(alert).toHaveTextContent("Failed to load"); + expect(alert).toHaveTextContent("Request timed out after 30s"); + expect(screen.queryByText("No people")).not.toBeInTheDocument(); + await userEvent.click(screen.getByRole("button", { name: "Retry" })); + expect(onRetry).toHaveBeenCalledTimes(1); + }); + + it("keeps showing rows it already has when a refetch fails", () => { + render(); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(screen.getByText("Alice")).toBeInTheDocument(); + }); +}); + describe("DataTable layout", () => { it("stretches the table to fill the container when resizing is on, so hidden columns leave no right-side gap", () => { render(); diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx index a94c4e34611..0c484de8e73 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx +++ b/ui/litellm-dashboard/src/components/shared/DataTable/DataTable.tsx @@ -26,6 +26,7 @@ import { type VisibilityState, } from "@tanstack/react-table"; import { SearchX } from "lucide-react"; +import { ErrorState } from "@/components/shared/ErrorState"; import * as React from "react"; import { Fragment, useEffect, useState } from "react"; @@ -599,6 +600,9 @@ export function DataTable(props: DataTableProps(props: DataTableProps ); } + if (isError && rows.length === 0) { + return ( + + + + ); + } if (rows.length === 0) { return ( diff --git a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts index 4529e3df164..787ea633492 100644 --- a/ui/litellm-dashboard/src/components/shared/DataTable/types.ts +++ b/ui/litellm-dashboard/src/components/shared/DataTable/types.ts @@ -28,6 +28,8 @@ export interface DataTableResolvedProps { isLoading?: boolean; isError?: boolean; + errorMessage?: React.ReactNode; + onRetry?: () => void; loadingMessage?: string; skeletonRowCount?: number; noDataMessage?: React.ReactNode; diff --git a/ui/litellm-dashboard/src/components/shared/ErrorState.tsx b/ui/litellm-dashboard/src/components/shared/ErrorState.tsx new file mode 100644 index 00000000000..140aaf211c9 --- /dev/null +++ b/ui/litellm-dashboard/src/components/shared/ErrorState.tsx @@ -0,0 +1,26 @@ +import { Button } from "@/components/ui/button"; +import { TriangleAlert } from "lucide-react"; +import * as React from "react"; + +interface ErrorStateProps { + title?: string; + message?: React.ReactNode; + onRetry?: () => void; +} + +export function ErrorState({ title = "Failed to load", message, onRetry }: ErrorStateProps) { + return ( +
+
+ +
+
{title}
+ {message ?
{message}
: null} + {onRetry ? ( + + ) : null} +
+ ); +} From 5e1e66d1ee87394ffff26d02d9c0e4fd5958c1da Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 17:39:17 -0700 Subject: [PATCH 031/235] feat(wandb): add Beam 1.0 and GLM-5.3 to the cost map (#47690) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 30 +++++++++++++++++++ model_prices_and_context_window.json | 30 +++++++++++++++++++ 2 files changed, 60 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 98fee42de3c..3d89af9d691 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76988,5 +76988,35 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "wandb/reflection/Beam-1.0-501B-A23B": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "wandb", + "max_input_tokens": 1049000, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wandb/zai-org/GLM-5.3": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "wandb", + "max_input_tokens": 1049000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 98fee42de3c..3d89af9d691 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76988,5 +76988,35 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true + }, + "wandb/reflection/Beam-1.0-501B-A23B": { + "cache_read_input_token_cost": 1e-07, + "input_cost_per_token": 1e-06, + "litellm_provider": "wandb", + "max_input_tokens": 1049000, + "mode": "chat", + "output_cost_per_token": 3e-06, + "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false + }, + "wandb/zai-org/GLM-5.3": { + "cache_read_input_token_cost": 1.9e-07, + "input_cost_per_token": 1.2e-06, + "litellm_provider": "wandb", + "max_input_tokens": 1049000, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference", + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": false } } From c8f7e0de7de3a2c3fee2a24a1bbc21c6abbb9d94 Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Tue, 20 Oct 2026 17:45:06 -0700 Subject: [PATCH 032/235] feat(model hub): show admin-defined custom tags and metadata from model_info (#46064) * feat(model hub): show admin-defined custom tags and metadata from model_info Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(router): move model hub custom field merging into router_utils Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(model hub): share the custom tag badge list between the column and details Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(model hub): cover public hub, patch, null, hostile, forwarding, and killed-worker paths for custom fields Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model hub): drop custom_metadata entries with non-string keys or non-finite numbers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(model hub): use isinstance narrowing instead of TypeGuard for custom_metadata values Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(model hub): build custom tags and metadata in the proxy instead of the router Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model hub): serve custom tags and metadata on model_group_alias rows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * remove docstring from model_group_custom_fields.py * style(model hub): fix lint and format in custom metadata tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(model hub): drop leading blank line in custom fields helper Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../model_group_custom_fields.py | 68 +++++ litellm/proxy/proxy_server.py | 19 +- .../model_management_endpoints.py | 3 + .../test_model_hub_custom_metadata.py | 276 ++++++++++++++++++ .../public_v1/test_model_hub.py | 8 +- tests/unit/proxy/test_proxy_utils.py | 106 ++++++- .../AIHub/ModelCustomMetadata.test.tsx | 29 ++ .../components/AIHub/ModelCustomMetadata.tsx | 48 +++ .../src/components/AIHub/ModelHubTable.tsx | 5 + .../AIHub/ModelHubTableColumns.test.tsx | 11 + .../components/AIHub/ModelHubTableColumns.tsx | 18 ++ .../components/PublicModelHubTableColumns.tsx | 10 + .../src/components/public_model_hub.tsx | 5 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 14 files changed, 604 insertions(+), 8 deletions(-) create mode 100644 litellm/proxy/management_helpers/model_group_custom_fields.py create mode 100644 tests/integration/management/test_model_hub_custom_metadata.py create mode 100644 ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.test.tsx create mode 100644 ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.tsx diff --git a/litellm/proxy/management_helpers/model_group_custom_fields.py b/litellm/proxy/management_helpers/model_group_custom_fields.py new file mode 100644 index 00000000000..2e94d47a952 --- /dev/null +++ b/litellm/proxy/management_helpers/model_group_custom_fields.py @@ -0,0 +1,68 @@ +import math +from collections.abc import Iterator, Mapping, Sequence +from types import MappingProxyType +from typing import Final, TypeAlias + +from pydantic import TypeAdapter + +CustomMetadataValue: TypeAlias = str | int | float | bool + +_OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(Mapping[object, object]) +_OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) + + +def _string_tags(tags: object) -> Iterator[str]: + if isinstance(tags, list): + yield from (tag for tag in _OBJECT_LIST_ADAPTER.validate_python(tags) if isinstance(tag, str)) + + +def _is_non_finite(value: object) -> bool: + return isinstance(value, float) and not math.isfinite(value) + + +def _scalar_metadata(metadata: object) -> Iterator[tuple[str, CustomMetadataValue]]: + if isinstance(metadata, dict): + yield from ( + (key, value) + for key, value in _OBJECT_MAPPING_ADAPTER.validate_python(metadata).items() + if isinstance(key, str) and isinstance(value, (str, int, float, bool)) and not _is_non_finite(value) + ) + + +def _custom_tag_values(model_infos: Sequence[Mapping[object, object]]) -> Iterator[str]: + for model_info in model_infos: + yield from _string_tags(model_info.get("custom_tags")) + + +def _custom_metadata_entries( + model_infos: Sequence[Mapping[object, object]], +) -> Iterator[tuple[str, CustomMetadataValue]]: + for model_info in model_infos: + yield from _scalar_metadata(model_info.get("custom_metadata")) + + +def _merged_custom_tags(model_infos: Sequence[Mapping[object, object]]) -> tuple[str, ...] | None: + return tuple(dict.fromkeys(_custom_tag_values(model_infos))) or None + + +def _merged_custom_metadata( + model_infos: Sequence[Mapping[object, object]], +) -> Mapping[str, CustomMetadataValue] | None: + entries: Final = tuple(_custom_metadata_entries(model_infos)) + first_seen: Final = MappingProxyType(dict(reversed(entries))) + return MappingProxyType({key: first_seen[key] for key in dict.fromkeys(key for key, _ in entries)}) or None + + +def _model_info(deployment: Mapping[str, object]) -> Mapping[object, object]: + model_info: Final = deployment.get("model_info") + return _OBJECT_MAPPING_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else MappingProxyType({}) + + +def model_group_custom_fields(deployments: Sequence[Mapping[str, object]]) -> Mapping[str, object]: + model_infos: Final = tuple(_model_info(deployment) for deployment in deployments) + return MappingProxyType( + { + "custom_tags": _merged_custom_tags(model_infos), + "custom_metadata": _merged_custom_metadata(model_infos), + } + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 967cae27990..8d6c9dd925b 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -166,6 +166,7 @@ from litellm.proxy.common_utils.token_count_settings import ( resolve_token_count_settings, ) from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog +from litellm.proxy.management_helpers.model_group_custom_fields import model_group_custom_fields from litellm.router_utils.access_windows import access_windows_config_error from litellm.router_utils.add_retry_fallback_headers import ( get_fallback_errors_from_headers, @@ -17747,10 +17748,7 @@ def get_model_group_info( ) -> list[ModelGroupInfoProxy]: model_groups: Final[list[ModelGroupInfoProxy]] = [] - unique_models: Final = [] - for model in all_models_str: - if model not in unique_models: - unique_models.append(model) + unique_models: Final = tuple(dict.fromkeys(all_models_str)) for model in unique_models: if model_group is not None and model_group != model: @@ -17759,7 +17757,18 @@ def get_model_group_info( _model_group_info = llm_router.get_model_group_info(model_group=model, team_id=team_id) if _model_group_info is not None: - model_groups.append(ModelGroupInfoProxy.model_validate(_model_group_info.model_dump())) + model_groups.append( + ModelGroupInfoProxy.model_validate( + { + **_model_group_info.model_dump(), + **model_group_custom_fields( + llm_router.get_model_list_of_routed_group( + resolve_model_group_alias(llm_router.model_group_alias, model) or model + ) + ), + } + ) + ) else: model_group_info = ModelGroupInfoProxy( model_group=model, diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index 3cc85a9427e..6a4160c3c50 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from datetime import datetime from typing import Any @@ -13,6 +14,8 @@ class ModelGroupInfoProxy(ModelGroupInfo): health_status: str | None = Field(default=None) health_response_time: float | None = Field(default=None) health_checked_at: str | None = Field(default=None) + custom_tags: tuple[str, ...] | None = None + custom_metadata: Mapping[str, str | int | float | bool] | None = None class UpdateUsefulLinksRequest(LiteLLMBaseModel): diff --git a/tests/integration/management/test_model_hub_custom_metadata.py b/tests/integration/management/test_model_hub_custom_metadata.py new file mode 100644 index 00000000000..6f007b6ebac --- /dev/null +++ b/tests/integration/management/test_model_hub_custom_metadata.py @@ -0,0 +1,276 @@ +import math +import uuid +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import pytest +import yaml +from pydantic import JsonValue + +from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS +from tests.integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from tests.integration._support.process import graceful_stop_seconds, owned_proxy_process + +LIFECYCLE_METADATA: Final = {"retirement_date": "2027-03-01", "ptu_max_tpm": 300000, "provisioned_ptu": 100} + + +def _model_group(gateway: Gateway, model_group: str) -> dict[str, JsonValue]: + data: Final = gateway.get("/model_group/info", params={"model_group": model_group})["data"] + assert isinstance(data, list) and len(data) == 1, data + return object_value(data[0]) + + +def _deployment(gateway: Gateway, scenario: Scenario, model_name: str, model_info: dict[str, JsonValue]) -> str: + created: Final = gateway.post( + "/model/new", + { + "model_name": model_name, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "integration-provider-key", + "api_base": f"{gateway.upstream_url}/v1", + }, + "model_info": model_info, + }, + ) + identity: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, identity) + return identity + + +def _custom_fields(row: dict[str, JsonValue]) -> tuple[JsonValue, JsonValue]: + assert "custom_tags" in row and "custom_metadata" in row, row + return row["custom_tags"], row["custom_metadata"] + + +def _public_row(gateway: Gateway, path: str, model_group: str) -> dict[str, JsonValue] | None: + response: Final = gateway.client.get(path) + assert response.status_code == 200, response.text + body: Final = response.json() + rows: Final = body if isinstance(body, list) else object_value(body)["data"] + assert isinstance(rows, list), response.text + return next((object_value(row) for row in rows if object_value(row).get("model_group") == model_group), None) + + +def test_model_group_info_serves_the_custom_tags_and_metadata_set_on_the_model(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model( + model_info={"custom_tags": ["deprecated", "ptu"], "custom_metadata": LIFECYCLE_METADATA} + ) + group: Final = _model_group(gateway, model) + assert group.get("custom_tags") == ["deprecated", "ptu"], group + assert group.get("custom_metadata") == LIFECYCLE_METADATA, group + + +def test_model_group_info_merges_custom_tags_and_metadata_across_the_groups_deployments(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model_name: Final = f"integration-{uuid.uuid4().hex}" + _deployment( + gateway, + scenario, + model_name, + {"custom_tags": ["ptu", "eu"], "custom_metadata": {"provisioned_ptu": 100, "region": "westeurope"}}, + ) + _deployment( + gateway, + scenario, + model_name, + { + "custom_tags": ["eu", "deprecated"], + "custom_metadata": {"provisioned_ptu": 50, "retirement_date": "2027-03-01"}, + }, + ) + group: Final = _model_group(gateway, model_name) + assert sorted(string_value(tag) for tag in group.get("custom_tags") or []) == ["deprecated", "eu", "ptu"], group + metadata: Final = object_value(group.get("custom_metadata")) + assert metadata.get("region") == "westeurope", metadata + assert metadata.get("retirement_date") == "2027-03-01", metadata + assert metadata.get("provisioned_ptu") in (100, 50), metadata + + +def test_model_group_info_ignores_custom_metadata_values_that_are_not_scalars(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model( + model_info={ + "custom_tags": ["ok", 7, {"nested": True}], + "custom_metadata": {"owner": "platform", "nested": {"a": 1}, "list": [1, 2]}, + } + ) + group: Final = _model_group(gateway, model) + assert group.get("custom_tags") == ["ok"], group + assert group.get("custom_metadata") == {"owner": "platform"}, group + + +@pytest.mark.parametrize("path", ["/public/model_hub", "/public/v1/model_hub"]) +def test_public_model_hub_serves_the_custom_tags_and_metadata_of_a_public_model_group( + gateway: Gateway, path: str +) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model( + model_info={"custom_tags": ["deprecated", "ptu"], "custom_metadata": LIFECYCLE_METADATA} + ) + gateway.post("/model_group/make_public", {"model_groups": [model]}) + row: Final = eventually( + lambda: _public_row(gateway, path, model), + lambda found: found is not None, + seconds=2 * PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, + ) + assert row is not None + assert _custom_fields(row) == (["deprecated", "ptu"], LIFECYCLE_METADATA), row + + +def test_patching_a_model_replaces_the_custom_tags_and_metadata_served_for_its_group(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model_name: Final = f"integration-{uuid.uuid4().hex}" + identity: Final = _deployment( + gateway, scenario, model_name, {"custom_tags": ["ptu"], "custom_metadata": LIFECYCLE_METADATA} + ) + patched: Final = gateway.request( + "PATCH", + f"/model/{identity}/update", + {"model_info": {"custom_tags": ["retiring"], "custom_metadata": {"retirement_date": "2027-06-01"}}}, + ) + assert patched.status_code == 200, patched.text + group: Final = eventually( + lambda: _model_group(gateway, model_name), + lambda found: found.get("custom_tags") == ["retiring"], + seconds=2 * PROXY_CONFIG_RELOAD_INTERVAL_SECONDS, + ) + assert _custom_fields(group) == (["retiring"], {"retirement_date": "2027-06-01"}), group + + +@pytest.mark.parametrize( + "model_info", + [ + pytest.param({"description": "plain model"}, id="missing"), + pytest.param({"custom_tags": [], "custom_metadata": {}}, id="empty"), + pytest.param({"custom_tags": None, "custom_metadata": None}, id="null"), + pytest.param({"custom_tags": "ptu", "custom_metadata": "retiring"}, id="wrong-shape"), + pytest.param( + {"custom_tags": [1, None, {"a": 1}], "custom_metadata": {"nested": {"a": 1}}}, id="no-valid-entry" + ), + ], +) +def test_model_group_info_serves_null_custom_fields_when_the_model_sets_none_usable( + gateway: Gateway, model_info: dict[str, JsonValue] +) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model(model_info=model_info) + group: Final = _model_group(gateway, model) + assert _custom_fields(group) == (None, None), group + + +def test_model_group_info_keeps_long_values_whole_and_serves_a_repeated_tag_once(gateway: Gateway) -> None: + long_value: Final = "x" * 5000 + with gateway.scenario() as scenario: + model: Final = scenario.model( + model_info={"custom_tags": ["ptu", "ptu", "eu", "ptu"], "custom_metadata": {"notes": long_value}} + ) + group: Final = _model_group(gateway, model) + assert _custom_fields(group) == (["ptu", "eu"], {"notes": long_value}), group + + +def test_custom_tags_and_metadata_are_never_forwarded_to_the_provider(gateway: Gateway) -> None: + marker: Final = f"custom-fields-{uuid.uuid4().hex}" + with ( + gateway.scenario() as scenario, + httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream, + ): + model: Final = scenario.model( + model_info={"custom_tags": ["deprecated"], "custom_metadata": {"retirement_date": "2027-03-01"}} + ) + reply: Final = gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]} + ) + assert reply.status_code == 200, reply.text + observed: Final = upstream.get("/__observations") + assert observed.status_code == 200, observed.text + bodies: Final = [ + str(object_value(value)["body"]) + for value in object_value(observed.json())["requests"] or [] + if marker in str(object_value(value)["body"]) + ] + assert len(bodies) == 1, bodies + assert "custom_tags" not in bodies[0] and "2027-03-01" not in bodies[0], bodies[0] + + +def _yaml_config( + directory: Path, + upstream_url: str, + model_name: str, + model_info: Mapping[str, object] = MappingProxyType( + {"custom_tags": ["deprecated", "ptu"], "custom_metadata": LIFECYCLE_METADATA} + ), + model_group_alias: Mapping[str, str] = MappingProxyType({}), +) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + deployment: Final = { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-4o-mini", "api_base": f"{upstream_url}/v1", "api_key": "yaml-key"}, + "model_info": dict(model_info), + } + path: Final = directory / "custom_fields.yaml" + router_settings: Final = {**config.get("router_settings", {}), "model_group_alias": dict(model_group_alias)} + path.write_text( + yaml.safe_dump( + {**config, "model_list": [*config["model_list"], deployment], "router_settings": router_settings} + ) + ) + return path + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +def test_yaml_custom_fields_survive_a_killed_worker_on_every_worker(gateway: Gateway, tmp_path: Path) -> None: + model_name: Final = f"integration-yaml-{uuid.uuid4().hex}" + config: Final = _yaml_config(tmp_path, gateway.upstream_url, model_name) + with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned: + victim: Final = min(owned.worker_pids()) + owned.kill_worker(victim) + with ThreadPoolExecutor(max_workers=8) as pool: + groups: Final = tuple(pool.map(lambda _: _model_group(owned.gateway, model_name), range(24))) + assert all(_custom_fields(group) == (["deprecated", "ptu"], LIFECYCLE_METADATA) for group in groups), groups + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +@pytest.mark.parametrize( + "unservable_metadata", + [ + pytest.param({2027: "retirement"}, id="non-string-key"), + pytest.param({"nan": math.nan, "inf": math.inf, "neg_inf": -math.inf}, id="non-finite-number"), + ], +) +def test_yaml_custom_metadata_that_json_cannot_carry_is_dropped_without_breaking_completions_or_the_hub( + gateway: Gateway, tmp_path: Path, unservable_metadata: Mapping[object, object] +) -> None: + model_name: Final = f"integration-yaml-{uuid.uuid4().hex}" + config: Final = _yaml_config( + tmp_path, + gateway.upstream_url, + model_name, + {"custom_tags": ["ptu"], "custom_metadata": {**unservable_metadata, "retirement_date": "2027-03-01"}}, + ) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + reply: Final = owned.gateway.request( + "POST", "/v1/chat/completions", {"model": model_name, "messages": [{"role": "user", "content": "hi"}]} + ) + assert reply.status_code == 200, reply.text + every_group: Final = owned.gateway.request("GET", "/model_group/info") + assert every_group.status_code == 200, every_group.text + group: Final = _model_group(owned.gateway, model_name) + assert _custom_fields(group) == (["ptu"], {"retirement_date": "2027-03-01"}), group + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +def test_an_alias_group_serves_the_custom_tags_and_metadata_of_the_group_it_points_to( + gateway: Gateway, tmp_path: Path +) -> None: + model_name: Final = f"integration-yaml-{uuid.uuid4().hex}" + alias: Final = f"{model_name}-alias" + config: Final = _yaml_config(tmp_path, gateway.upstream_url, model_name, model_group_alias={alias: model_name}) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + group: Final = _model_group(owned.gateway, alias) + assert _custom_fields(group) == (["deprecated", "ptu"], LIFECYCLE_METADATA), group diff --git a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py index 43cdebd54b6..ddb4b08ee9e 100644 --- a/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py +++ b/tests/unit/proxy/public_endpoints/public_v1/test_model_hub.py @@ -10,7 +10,7 @@ from fastapi.testclient import TestClient import litellm from litellm.proxy._types import LiteLLMRoutes from litellm.proxy.proxy_server import app -from litellm.types.router import ModelGroupInfo +from litellm.types.router import DeploymentTypedDict, ModelGroupInfo from tests._master_key import MASTER_KEY client = TestClient(app) @@ -21,13 +21,17 @@ LEGACY_MODEL_HUB_PATH = "/public/model_hub" @dataclass(frozen=True, slots=True) class _FakeRouter: - """Stands in for the running Router: `_get_model_group_info` only ever asks it this.""" + """Stands in for the running Router: `_get_model_group_info` only ever asks it these.""" infos: Mapping[str, ModelGroupInfo] + model_group_alias: Mapping[str, str] = MappingProxyType({}) def get_model_group_info(self, model_group: str, team_id: str | None = None) -> ModelGroupInfo | None: return self.infos.get(model_group) + def get_model_list_of_routed_group(self, model_group: str) -> list[DeploymentTypedDict]: + return [] + def _info( name: str, diff --git a/tests/unit/proxy/test_proxy_utils.py b/tests/unit/proxy/test_proxy_utils.py index 02f31bf07c6..c77d7ce8512 100644 --- a/tests/unit/proxy/test_proxy_utils.py +++ b/tests/unit/proxy/test_proxy_utils.py @@ -1,7 +1,7 @@ import asyncio import json from datetime import datetime -from typing import Any, Dict, Final, List, Optional, Union +from typing import Any, Dict, Final, List, Optional, TypeAlias, Union from unittest.mock import Mock import pytest @@ -24,6 +24,7 @@ from litellm.proxy.litellm_pre_call_utils import ( get_dynamic_logging_metadata, add_litellm_data_to_request, ) +from litellm.router import Router from pydantic import ValidationError pytestmark = pytest.mark.xdist_group("proxy_heavy") @@ -1602,6 +1603,109 @@ def test_get_model_group_info(): assert len(model_list) == 1 +def _custom_metadata_router(*model_infos: dict[object, object]) -> Router: + return Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": f"fake-{index}"}, + "model_info": model_info, + } + for index, model_info in enumerate(model_infos) + ] + ) + + +_CustomFieldsCase: TypeAlias = tuple[ + tuple[dict[object, object], ...], tuple[str, ...] | None, dict[str, str | int | float | bool] | None +] +_CUSTOM_FIELDS_CASES: Final[tuple[_CustomFieldsCase, ...]] = ( + ((), None, None), + (({},), None, None), + ( + ({"custom_tags": ["deprecated", "ptu"], "custom_metadata": {"retirement_date": "2027-03-01"}},), + ("deprecated", "ptu"), + {"retirement_date": "2027-03-01"}, + ), + ( + ({}, {"custom_tags": ["ptu"], "custom_metadata": {"provisioned_ptu": 100, "gsu": False}}), + ("ptu",), + {"provisioned_ptu": 100, "gsu": False}, + ), + ( + ( + {"custom_tags": ["ptu", "eu"], "custom_metadata": {"provisioned_ptu": 100, "region": "eu"}}, + {"custom_tags": ["eu", "deprecated"], "custom_metadata": {"provisioned_ptu": 50, "max_tpm": 1.5}}, + ), + ("ptu", "eu", "deprecated"), + {"provisioned_ptu": 100, "region": "eu", "max_tpm": 1.5}, + ), + ( + ({"custom_tags": ["ok", 7, None, {"x": 1}], "custom_metadata": {"owner": "ml", "nested": {"a": 1}}},), + ("ok",), + {"owner": "ml"}, + ), + (({"custom_tags": "ptu", "custom_metadata": ["not", "a", "map"]},), None, None), + ( + ( + { + "custom_metadata": { + 2027: "retirement", + "nan": float("nan"), + "inf": float("inf"), + "neg_inf": float("-inf"), + "max_tpm": 1.5, + } + }, + ), + None, + {"max_tpm": 1.5}, + ), +) + + +@pytest.mark.parametrize(("model_infos", "expected_tags", "expected_metadata"), _CUSTOM_FIELDS_CASES) +def test_get_model_group_info_merges_custom_tags_and_metadata_across_deployments( + model_infos: tuple[dict[object, object], ...], + expected_tags: tuple[str, ...] | None, + expected_metadata: dict[str, str | int | float | bool] | None, +) -> None: + from litellm.proxy.proxy_server import get_model_group_info + + router: Final = _custom_metadata_router({"description": "first"}, *model_infos) + + (group,) = get_model_group_info(llm_router=router, all_models_str=["gpt-4"], model_group="gpt-4") + + assert (group.custom_tags, group.custom_metadata) == (expected_tags, expected_metadata) + + +@pytest.mark.parametrize( + "alias_target", + [pytest.param("gpt-4", id="string"), pytest.param({"model": "gpt-4", "hidden": False}, id="item")], +) +def test_get_model_group_info_serves_the_target_groups_custom_fields_on_an_alias_row( + alias_target: str | dict[str, object], +) -> None: + from litellm.proxy.proxy_server import get_model_group_info + + router: Final = Router( + model_list=[ + { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4", "api_key": "fake"}, + "model_info": {"custom_tags": ["deprecated"], "custom_metadata": {"retirement_date": "2027-03-01"}}, + } + ], + model_group_alias={"gpt-alias": alias_target}, + ) + + groups: Final = get_model_group_info(llm_router=router, all_models_str=["gpt-4", "gpt-alias"], model_group=None) + + assert {group.model_group: (group.custom_tags, group.custom_metadata) for group in groups} == { + model_group: (("deprecated",), {"retirement_date": "2027-03-01"}) for model_group in ("gpt-4", "gpt-alias") + } + + @pytest.fixture def mock_team_data(): return [ diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.test.tsx new file mode 100644 index 00000000000..49681154de4 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.test.tsx @@ -0,0 +1,29 @@ +import { render, screen } from "@testing-library/react"; +import { describe, expect, it } from "vitest"; +import { ModelCustomMetadata } from "./ModelCustomMetadata"; + +describe("ModelCustomMetadata", () => { + it("renders each tag and every metadata entry with its value", () => { + render( + , + ); + expect(screen.getByText("Tags:")).toBeInTheDocument(); + expect(screen.getByText("deprecated")).toBeInTheDocument(); + expect(screen.getByText("ptu")).toBeInTheDocument(); + const terms = screen.getAllByRole("term").map((term) => term.textContent); + const definitions = screen.getAllByRole("definition").map((definition) => definition.textContent); + expect(terms.map((term, index) => [term, definitions[index]])).toEqual([ + ["retirement_date", "2027-03-01"], + ["provisioned_ptu", "100"], + ["gsu", "false"], + ]); + }); + + it("renders nothing when the model has no tags or metadata", () => { + const { container } = render(); + expect(container).toBeEmptyDOMElement(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.tsx new file mode 100644 index 00000000000..be2f509b646 --- /dev/null +++ b/ui/litellm-dashboard/src/components/AIHub/ModelCustomMetadata.tsx @@ -0,0 +1,48 @@ +import { Badge } from "@/components/ui/badge"; + +export type ModelCustomMetadataValue = string | number | boolean; + +interface ModelCustomMetadataProps { + customTags?: string[] | null; + customMetadata?: Record | null; +} + +export function ModelCustomTags({ tags, className = "" }: { tags: string[]; className?: string }) { + return ( +
+ {tags.map((tag) => ( + + {tag} + + ))} +
+ ); +} + +export function ModelCustomMetadata({ customTags, customMetadata }: ModelCustomMetadataProps) { + const tags = customTags ?? []; + const entries = Object.entries(customMetadata ?? {}); + return ( + <> + {tags.length > 0 && ( +
+

Tags:

+ +
+ )} + {entries.length > 0 && ( +
+

Custom Metadata:

+
+ {entries.map(([key, value]) => ( +
+
{key}
+
{String(value)}
+
+ ))} +
+
+ )} + + ); +} diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx index fb665eb69c2..b5c2a3e3b05 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx @@ -54,6 +54,7 @@ import { isDecisionMode, } from "@/lib/decisionModels"; import DecisionModelGarden from "@/components/AIHub/DecisionModelGarden"; +import { ModelCustomMetadata } from "./ModelCustomMetadata"; interface ModelHubTableProps { accessToken: string | null; @@ -719,6 +720,10 @@ const ModelHubTable: React.FC = ({

{selectedModel.description}

)} +
diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.test.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.test.tsx index dbe0371241a..f91eea4e301 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.test.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.test.tsx @@ -61,6 +61,17 @@ describe("getModelHubTableColumns", () => { expect(screen.getByRole("cell", { name: "gpt-4o" })).not.toHaveClass("text-right"); }); + it("shows each custom tag as a badge and a dash when the model has none", () => { + renderTable([ + { ...mockModel, custom_tags: ["deprecated", "ptu"] }, + { ...mockModel, model_group: "untagged", custom_tags: null }, + ]); + expect(screen.getByRole("columnheader", { name: "Tags" })).toBeInTheDocument(); + const [taggedRow, untaggedRow] = screen.getAllByRole("row").slice(1); + expect(taggedRow).toHaveTextContent("deprecatedptu"); + expect(untaggedRow).not.toHaveTextContent("deprecated"); + }); + it("shows capability badges only for supported features", () => { renderTable([mockModel]); expect(screen.getByText("Vision")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx index 1a5f8889c93..e2dfdc67aab 100644 --- a/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/AIHub/ModelHubTableColumns.tsx @@ -17,6 +17,8 @@ import { cn } from "@/lib/cva.config"; import { displayMode } from "@/lib/decisionModels"; import { copyToClipboard } from "@/utils/dataUtils"; +import { ModelCustomTags, type ModelCustomMetadataValue } from "./ModelCustomMetadata"; + export interface ModelHubData { model_group: string; providers: string[]; @@ -33,6 +35,8 @@ export interface ModelHubData { supported_openai_params?: string[]; is_public_model_group: boolean; description?: string; + custom_tags?: string[] | null; + custom_metadata?: Record | null; [key: string]: any; } @@ -210,6 +214,20 @@ export const getModelHubTableColumns = ({ onModelClick }: ModelHubTableColumnsDe ); }, }, + { + id: "custom_tags", + meta: { title: "Tags", skeleton: "chips" }, + header: "Tags", + size: 160, + enableSorting: false, + cell: ({ row }) => { + const tags = row.original.custom_tags ?? []; + if (tags.length === 0) { + return -; + } + return ; + }, + }, { id: "is_public_model_group", accessorKey: "is_public_model_group", diff --git a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx index 0288c58e31e..94eafdea86c 100644 --- a/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx +++ b/ui/litellm-dashboard/src/components/PublicModelHubTableColumns.tsx @@ -8,6 +8,7 @@ import { Badge } from "@/components/ui/badge"; import { getProviderLogoAndName } from "@/components/provider_info_helpers"; import { PUBLIC_MODEL_HUB_SORTABLE_FIELDS } from "@/components/publicModelHub/publicModelHubFilters"; import { displayMode } from "@/lib/decisionModels"; +import type { ModelCustomMetadataValue } from "@/components/AIHub/ModelCustomMetadata"; export interface ModelGroupInfo { model_group: string; @@ -27,6 +28,8 @@ export interface ModelGroupInfo { health_response_time?: number; health_checked_at?: string; description?: string; + custom_tags?: string[] | null; + custom_metadata?: Record | null; [key: string]: any; } @@ -260,6 +263,13 @@ export const getPublicModelHubColumns = ({ onModelClick }: PublicModelHubColumns return ; }, }, + { + id: "custom_tags", + meta: { title: "Tags", skeleton: "chips" }, + header: "Tags", + size: 140, + cell: ({ row }) => , + }, { id: "health_status", accessorKey: "health_status", diff --git a/ui/litellm-dashboard/src/components/public_model_hub.tsx b/ui/litellm-dashboard/src/components/public_model_hub.tsx index 78adb7e4d40..23879a1d23e 100644 --- a/ui/litellm-dashboard/src/components/public_model_hub.tsx +++ b/ui/litellm-dashboard/src/components/public_model_hub.tsx @@ -53,6 +53,7 @@ import { getProviderLogoAndName } from "./provider_info_helpers"; import { displayMode, isDecisionMode } from "@/lib/decisionModels"; import { modelUsageExample } from "@/components/AIHub/modelUsageExample"; import { filterBySearchTerm, rankBySearchRelevance } from "@/utils/searchUtils"; +import { ModelCustomMetadata } from "@/components/AIHub/ModelCustomMetadata"; interface PublicModelHubProps { accessToken?: string | null; @@ -749,6 +750,10 @@ const PublicModelHub: React.FC = ({ accessToken, isEmbedded

{selectedModel.description}

)} + {/* Wildcard Routing Note */} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 244141ec076..2b174a1141a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -38374,6 +38374,12 @@ export interface components { ModelGroupInfoProxy: { /** Configurable Clientside Auth Params */ configurable_clientside_auth_params?: (string | components["schemas"]["ConfigurableClientsideParamsCustomAuth-Output"])[] | null; + /** Custom Metadata */ + custom_metadata?: { + [key: string]: string | number | boolean; + } | null; + /** Custom Tags */ + custom_tags?: string[] | null; /** Description */ description?: string | null; /** Health Checked At */ From f24036c99d58bca5830b1f833d67645a87102920 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 17:47:04 -0700 Subject: [PATCH 033/235] test(perf): emit the database upgrade run's perf rows and upgrade timings as JUnit properties (#47696) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/setup/upgrade.py | 12 ++++++++ tests/e2e/perf/test_database_upgrade.py | 33 +++++++++++++++----- tests/e2e_harness/perf/test_upgrade.py | 41 ++++++++++++++++++++++++- 3 files changed, 77 insertions(+), 9 deletions(-) diff --git a/tests/e2e/perf/setup/upgrade.py b/tests/e2e/perf/setup/upgrade.py index 37e5c7bc904..36efc16e1fe 100644 --- a/tests/e2e/perf/setup/upgrade.py +++ b/tests/e2e/perf/setup/upgrade.py @@ -199,6 +199,18 @@ class UpgradeMonitor: assert not state.pending_indexes or self.overlap_writes > 0, "No persisted requests observed during index work" +def upgrade_properties(state: UpgradeState, monitor: UpgradeMonitor) -> tuple[tuple[str, float], ...]: + """The upgrade's own numbers under the `perf_` prefix the dashboard ingests, next to the perf rows.""" + return ( + ("perf_upgrade_schema_seconds", state.schema_seconds), + ("perf_upgrade_index_seconds", state.index_seconds), + ("perf_upgrade_overlap_writes", float(monitor.overlap_writes)), + ("perf_upgrade_max_lock_waiters", float(monitor.max_lock_waiters)), + ("perf_upgrade_pending_indexes", float(len(state.pending_indexes))), + ("perf_upgrade_unmatched_requests", float(len(monitor.pending))), + ) + + def monitor_for(connection: psycopg.Connection[tuple[object, ...]], urls: tuple[str, ...]) -> UpgradeMonitor: return UpgradeMonitor( state=lambda: read_state(connection), diff --git a/tests/e2e/perf/test_database_upgrade.py b/tests/e2e/perf/test_database_upgrade.py index 8079aa0274d..fba6af30014 100644 --- a/tests/e2e/perf/test_database_upgrade.py +++ b/tests/e2e/perf/test_database_upgrade.py @@ -12,19 +12,29 @@ from lifecycle import ResourceManager from memory_readings import read_rss_everywhere from proxy_client import ProxyClient from setup.datastore import read_db_counters, read_redis_counters -from setup.observe import Observation, evaluate, read_mock_requests, render, reset_mock, set_mock_latency +from setup.observe import ( + Observation, + evaluate, + junit_properties, + read_mock_requests, + render, + reset_mock, + set_mock_latency, +) from setup.plan import build_plan from setup.podcpu import gateway_cpu_sampler from setup.provision import MOCK_PROVIDERS, build_setup, mock_url from setup.run import run_plan -from setup.upgrade import assert_finished, monitor_for, upgrade_case +from setup.upgrade import assert_finished, monitor_for, upgrade_case, upgrade_properties from setup.upgrade_worker import connect, read_state pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack] @pytest.mark.skipif(os.environ.get("E2E_DATABASE_UPGRADE") != "1", reason="Dedicated disposable snapshot lane only") -def test_database_upgrade(proxy: ProxyClient, idp: Keycloak, resources: ResourceManager) -> None: +def test_database_upgrade( + proxy: ProxyClient, idp: Keycloak, resources: ResourceManager, request: pytest.FixtureRequest +) -> None: case: Final = upgrade_case() urls: Final = tuple(mock_url(provider) for provider in MOCK_PROVIDERS) for url in urls: @@ -52,12 +62,7 @@ def test_database_upgrade(proxy: ProxyClient, idp: Keycloak, resources: Resource f"successful_requests={monitor.successes} persisted_requests={monitor.persisted_count} " f"unmatched_requests={len(monitor.pending)}\n" ) - monitor.assert_complete(state) - assert_finished(connection, state) rss_after: Final = read_rss_everywhere(proxy) - assert {(r.replica, r.hostname) for r in rss_before.readings} == { - (r.replica, r.hostname) for r in rss_after.readings - }, "Gateway pods replaced during the upgrade" observed: Final = Observation( records=outcome.records, mock_requests=tuple(seen for url in urls for seen in read_mock_requests(url)), @@ -73,6 +78,13 @@ def test_database_upgrade(proxy: ProxyClient, idp: Keycloak, resources: Resource gateway_cpu_seconds=cpu.seconds, ) rows, exceeded = evaluate(case, observed) + item: Final = request.node # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # pytest: untyped + assert isinstance(item, pytest.Item) + item.user_properties[:] = [ + entry for entry in item.user_properties if not entry[0].startswith("perf_") + ] # rebind-ok: a --reruns retry replaces the first attempt's numbers instead of appending a second set + item.user_properties.extend(junit_properties(rows)) + item.user_properties.extend(upgrade_properties(state, monitor)) _ = sys.stdout.write(render(case, observed, rows) + "\n") _ = sys.stdout.write( f"schema_seconds={state.schema_seconds:.3f} index_seconds={state.index_seconds:.3f} " @@ -82,4 +94,9 @@ def test_database_upgrade(proxy: ProxyClient, idp: Keycloak, resources: Resource artifact: Final = os.environ.get("E2E_ARTIFACT_DIR") if artifact: _ = (Path(artifact) / "database-upgrade.json").write_text(state.model_dump_json(), encoding="utf-8") + monitor.assert_complete(state) + assert_finished(connection, state) + assert {(r.replica, r.hostname) for r in rss_before.readings} == { + (r.replica, r.hostname) for r in rss_after.readings + }, "Gateway pods replaced during the upgrade" assert not exceeded, "\n".join(exceeded) diff --git a/tests/e2e_harness/perf/test_upgrade.py b/tests/e2e_harness/perf/test_upgrade.py index 1f2bc8c46cb..9a6c9d2af14 100644 --- a/tests/e2e_harness/perf/test_upgrade.py +++ b/tests/e2e_harness/perf/test_upgrade.py @@ -7,7 +7,14 @@ import pytest from cases.small_bodies_5rps import CASE from memory_readings import RssCapture, RssReading from setup.observe import Observation, evaluate -from setup.upgrade import MAX_TRAFFIC_SECONDS, UpgradeMonitor, matched_response_ids, postgres_failures, upgrade_case +from setup.upgrade import ( + MAX_TRAFFIC_SECONDS, + UpgradeMonitor, + matched_response_ids, + postgres_failures, + upgrade_case, + upgrade_properties, +) from setup.upgrade_worker import UpgradeState from setup.wire import PlanProgress, WireRecord, response_id @@ -332,3 +339,35 @@ def test_upgrade_verdict_uses_regular_limits_without_cpu_or_datastore_samples( rows, exceeded = evaluate(upgrade_case(), observed) assert tuple(row.label for row in rows if row.over or row.missing) == expected assert len(exceeded) == len(expected) + + +def test_upgrade_properties_carry_the_upgrade_timings_and_what_the_monitor_saw() -> None: + state: Final = UpgradeState( + phase="complete", + started_at=0, + schema_seconds=12.5, + index_seconds=240.25, + required_indexes=("idx_a", "idx_b"), + required_migrations=("20260101_a",), + pending_indexes=("idx_b",), + baseline_rows={}, + ) + monitor: Final = UpgradeMonitor( + state=lambda: state, + check_metrics=lambda: None, + read_activity=lambda: ([], 0), + read_persisted=lambda records: frozenset(), + start_indexes=lambda: None, + ) + monitor.overlap_writes = 37 + monitor.max_lock_waiters = 3 + monitor.pending = (record(0), record(1)) + + assert dict(upgrade_properties(state, monitor)) == { + "perf_upgrade_schema_seconds": 12.5, + "perf_upgrade_index_seconds": 240.25, + "perf_upgrade_overlap_writes": 37, + "perf_upgrade_max_lock_waiters": 3, + "perf_upgrade_pending_indexes": 1, + "perf_upgrade_unmatched_requests": 2, + } From 3c28f4bf1e049222340e560bc6d59b7a9e43036c Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 20 Oct 2026 17:58:05 -0700 Subject: [PATCH 034/235] fix(ui): show the budget spend limit and rate limits without a collapsed section (#47662) * fix(ui): show the budget spend limit and rate limits without a collapsed section The create and edit budget dialogs hid max budget and reset period behind a collapsed Optional Settings section that sat 80px below the last field, and dropped both values from the request unless the section was open on submit Both dialogs now share BudgetFormFields. Budget ID comes first, then the spend limit as one row ("$500 per month"), then the three rate limits as short fields beside their labels. A secondary Optional badge marks the spend limit and the rate limits group, so people can still tell those fields are optional. Every field is always submitted, and the edit dialog sends the existing spend limit and reset period back unchanged * fix(ui): send only changed fields when editing a budget Saving the edit budget dialog resent every field, so an unrelated rate limit edit rounded a stored sub-cent spend limit to 2 decimals (0.001 became 0) and resent the reset period, which makes the proxy recompute the next reset for custom durations. The dialog now sends the budget ID plus the fields the user changed, and the proxy keeps everything else. Also updates the budgets Playwright spec for the new field labels and the removed Optional Settings section. * fix(ui): track budget edit changes from render so a reverted field cannot hide a later one react-hook-form's formState is a render-time snapshot. Reading dirtyFields only inside the submit handler meant that after a field was changed and changed back, a later edit to another field was missing from the first save. Reading dirtyFields during render keeps the snapshot current. * refactor(ui): rename the budget components to PascalCase Renames budget_modal, edit_budget_modal and budget_panel (and their tests) to BudgetModal, EditBudgetModal and BudgetPanel, and drops their filename-pascal-case suppressions. --- tests/e2e/ui/tests/budgets/budgets.spec.ts | 17 +- ui/litellm-dashboard/eslint-suppressions.json | 15 -- .../budgets/_components/BudgetFormFields.tsx | 154 +++++++++++++++ .../BudgetModal.integration.test.tsx | 92 +++++++++ .../budgets/_components/BudgetModal.tsx | 56 ++++++ ...et_panel.test.tsx => BudgetPanel.test.tsx} | 2 +- .../{budget_panel.tsx => BudgetPanel.tsx} | 4 +- .../EditBudgetModal.integration.test.tsx | 111 +++++++++++ .../budgets/_components/EditBudgetModal.tsx | 74 +++++++ .../budget_modal.integration.test.tsx | 138 ------------- .../budgets/_components/budget_modal.tsx | 181 ----------------- .../edit_budget_modal.integration.test.tsx | 112 ----------- .../budgets/_components/edit_budget_modal.tsx | 187 ------------------ .../src/app/(dashboard)/budgets/page.tsx | 2 +- 14 files changed, 497 insertions(+), 648 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetFormFields.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.tsx rename ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/{budget_panel.test.tsx => BudgetPanel.test.tsx} (99%) rename ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/{budget_panel.tsx => BudgetPanel.tsx} (98%) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.integration.test.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.integration.test.tsx delete mode 100644 ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx diff --git a/tests/e2e/ui/tests/budgets/budgets.spec.ts b/tests/e2e/ui/tests/budgets/budgets.spec.ts index 89691c05605..866c7688426 100644 --- a/tests/e2e/ui/tests/budgets/budgets.spec.ts +++ b/tests/e2e/ui/tests/budgets/budgets.spec.ts @@ -60,13 +60,11 @@ test.describe("Budgets", () => { await expect(modal).toBeVisible({ timeout: 10_000 }); await modal.getByRole("textbox", { name: "Budget ID" }).fill(budgetId); - await modal.getByRole("spinbutton", { name: "Max Tokens per minute" }).fill("5000"); - await modal.getByRole("spinbutton", { name: "Max Requests per minute" }).fill("60"); - - await modal.getByRole("button", { name: "Optional Settings" }).click(); - await modal.getByRole("spinbutton", { name: "Max Budget (USD)" }).fill("25.5"); - await modal.getByRole("combobox", { name: "Reset Budget" }).click(); - await page.getByRole("option", { name: "weekly" }).click(); + await modal.getByRole("spinbutton", { name: "Tokens per minute" }).fill("5000"); + await modal.getByRole("spinbutton", { name: "Requests per minute" }).fill("60"); + await modal.getByRole("spinbutton", { name: "Spend limit" }).fill("25.5"); + await modal.getByRole("combobox", { name: "Reset period" }).click(); + await page.getByRole("option", { name: "per week" }).click(); await modal.getByRole("button", { name: "Create Budget" }).click(); await expect(modal).not.toBeVisible({ timeout: 10_000 }); @@ -100,15 +98,12 @@ test.describe("Budgets", () => { const modal = page.getByRole("dialog", { name: "Edit Budget" }); await expect(modal).toBeVisible({ timeout: 10_000 }); - await modal.getByRole("button", { name: "Optional Settings" }).click(); - await modal.getByRole("spinbutton", { name: "Max Budget (USD)" }).fill("99"); + await modal.getByRole("spinbutton", { name: "Spend limit" }).fill("99"); await modal.getByRole("button", { name: "Save", exact: true }).click(); await expect(modal).not.toBeVisible({ timeout: 10_000 }); await expect(page.getByRole("row").filter({ hasText: budgetId })).toContainText("$99.00", { timeout: 10_000 }); - // Not hypothetical: the edit form posts the whole budget, so a field it fails to - // seed from the existing row goes to the server as null and silently clears. const stored = await findBudget(page, budgetId); expect(stored?.max_budget, "spend cap raised").toBe(99); expect(stored?.tpm_limit, "TPM limit untouched by a spend-cap edit").toBe(1000); diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 423287e7c0f..4802aa057c6 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -73,21 +73,6 @@ "count": 1 } }, - "src/app/(dashboard)/budgets/_components/budget_modal.tsx": { - "local/filename-pascal-case": { - "count": 1 - } - }, - "src/app/(dashboard)/budgets/_components/budget_panel.tsx": { - "local/filename-pascal-case": { - "count": 1 - } - }, - "src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx": { - "local/filename-pascal-case": { - "count": 1 - } - }, "src/app/(dashboard)/caching/_components/cache_dashboard.tsx": { "local/filename-pascal-case": { "count": 1 diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetFormFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetFormFields.tsx new file mode 100644 index 00000000000..9ffd23884b4 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetFormFields.tsx @@ -0,0 +1,154 @@ +import React, { useId } from "react"; +import { Controller } from "react-hook-form"; +import { z } from "zod"; +import { FormField } from "@/components/shared/form/FormField"; +import { Badge } from "@/components/ui/badge"; +import { ButtonGroup } from "@/components/ui/button-group"; +import { + Field, + FieldDescription, + FieldGroup, + FieldLabel, + FieldLegend, + FieldSeparator, + FieldSet, +} from "@/components/ui/field"; +import { Input } from "@/components/ui/input"; +import { InputGroup, InputGroupAddon, InputGroupInput, InputGroupText } from "@/components/ui/input-group"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { useZodForm } from "@/lib/forms/useZodForm"; + +const budgetShape = { + budget_id: z.string().min(1, "Enter a budget ID"), + tpm_limit: z.number().nullish(), + rpm_limit: z.number().nullish(), + tpd_limit: z.number().nullish(), + max_budget: z.number().nullish(), + budget_duration: z.string().nullish(), +}; + +const budgetSchema = z.object(budgetShape); + +export type BudgetFormValues = z.output; + +export const useBudgetForm = (defaultValues: z.input) => + useZodForm(budgetSchema, { defaultValues }); + +type BudgetForm = ReturnType; +type NumberFieldName = "tpm_limit" | "rpm_limit" | "tpd_limit" | "max_budget"; + +const NO_RESET = "never"; + +const RESET_OPTIONS = [ + { value: NO_RESET, label: "in total" }, + { value: "24h", label: "per day" }, + { value: "7d", label: "per week" }, + { value: "30d", label: "per month" }, +]; + +const numberProps = void }>({ + value, + onChange, + ...field +}: T) => ({ + ...field, + type: "number", + value: value ?? "", + placeholder: "No limit", + onChange: (event: React.ChangeEvent) => + onChange(event.target.value === "" ? null : event.target.valueAsNumber), +}); + +const OptionalBadge = () => Optional; + +function SpendLimitField({ form }: { form: BudgetForm }) { + const amountId = useId(); + return ( + + + Spend limit + + + + ( + + + $ + + + + )} + /> + ( + + )} + /> + + + ); +} + +function RateLimitField({ form, name, label }: { form: BudgetForm; name: NumberFieldName; label: string }) { + return ( + + {(control) => } + + ); +} + +export function BudgetFormFields({ form, isEditing }: { form: BudgetForm; isEditing: boolean }) { + return ( + + + {({ ref, ...field }) => } + + + +
+ + Rate limits + + + + + + + Batches count against the daily limit instead of the per-minute ones. Provider rate limits still apply. + +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.integration.test.tsx new file mode 100644 index 00000000000..0770150ab91 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.integration.test.tsx @@ -0,0 +1,92 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import BudgetModal from "./BudgetModal"; +import { chooseSelectOption } from "../../../../../tests/test-utils"; + +const { createMock } = vi.hoisted(() => ({ createMock: vi.fn() })); + +vi.mock("@/app/(dashboard)/hooks/budgets/useBudgets", () => ({ + useCreateBudget: () => ({ mutateAsync: createMock }), +})); + +const renderModal = () => render(); + +const create = async (user: ReturnType) => + user.click(screen.getByRole("button", { name: "Create budget" })); + +const resetPeriod = () => screen.getByRole("combobox", { name: "Reset period" }); + +describe("BudgetModal", () => { + beforeEach(() => { + vi.clearAllMocks(); + createMock.mockResolvedValue(undefined); + }); + + it("submits the spend limit and reset period alongside the rate limits", async () => { + const user = userEvent.setup(); + renderModal(); + + expect(screen.getByLabelText(/^Spend limit/)).toBeVisible(); + expect(resetPeriod()).toBeVisible(); + fireEvent.change(screen.getByLabelText("Budget ID"), { target: { value: "budget-alpha" } }); + fireEvent.change(screen.getByLabelText(/^Spend limit/), { target: { value: "42.567" } }); + await chooseSelectOption(user, resetPeriod(), "per month"); + fireEvent.change(screen.getByLabelText("Tokens per minute"), { target: { value: "500" } }); + fireEvent.change(screen.getByLabelText("Requests per minute"), { target: { value: "7" } }); + fireEvent.change(screen.getByLabelText("Batch tokens per day"), { target: { value: "90000" } }); + await create(user); + + await waitFor(() => expect(createMock).toHaveBeenCalledTimes(1)); + const expected = { + budget_id: "budget-alpha", + max_budget: 42.57, + budget_duration: "30d", + tpm_limit: 500, + rpm_limit: 7, + tpd_limit: 90000, + }; + expect(createMock.mock.calls[0][0]).toEqual(expected); + }); + + it("submits a spend limit that never resets as a null reset period", async () => { + const user = userEvent.setup(); + renderModal(); + + fireEvent.change(screen.getByLabelText("Budget ID"), { target: { value: "budget-alpha" } }); + fireEvent.change(screen.getByLabelText(/^Spend limit/), { target: { value: "10" } }); + expect(resetPeriod()).toHaveTextContent("in total"); + await chooseSelectOption(user, resetPeriod(), "per week"); + await chooseSelectOption(user, resetPeriod(), "in total"); + await create(user); + + await waitFor(() => expect(createMock).toHaveBeenCalledTimes(1)); + expect(createMock.mock.calls[0][0]).toEqual({ budget_id: "budget-alpha", max_budget: 10, budget_duration: null }); + }); + + it("submits a cleared number field as null", async () => { + const user = userEvent.setup(); + renderModal(); + + fireEvent.change(screen.getByLabelText("Budget ID"), { target: { value: "budget-alpha" } }); + fireEvent.change(screen.getByLabelText("Tokens per minute"), { target: { value: "5" } }); + await user.clear(screen.getByLabelText("Tokens per minute")); + await create(user); + + await waitFor(() => expect(createMock).toHaveBeenCalledTimes(1)); + expect(createMock.mock.calls[0][0]).toEqual({ budget_id: "budget-alpha", tpm_limit: null }); + }); + + it("blocks submit while Budget ID is empty", async () => { + const user = userEvent.setup(); + renderModal(); + + fireEvent.change(screen.getByLabelText("Tokens per minute"), { target: { value: "5" } }); + await create(user); + + await waitFor(() => expect(screen.getByLabelText("Budget ID")).toHaveAttribute("aria-invalid", "true")); + expect(createMock).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.tsx new file mode 100644 index 00000000000..bd10837640d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetModal.tsx @@ -0,0 +1,56 @@ +import React, { useId } from "react"; +import { useCreateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import { applyBudgetPrecision } from "./budgetPrecision"; +import { BudgetFormFields, type BudgetFormValues, useBudgetForm } from "./BudgetFormFields"; +import { toast } from "@/lib/toast"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogClose, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; + +interface BudgetModalProps { + isModalVisible: boolean; + setIsModalVisible: React.Dispatch>; +} +const BudgetModal: React.FC = ({ isModalVisible, setIsModalVisible }) => { + const formId = useId(); + const form = useBudgetForm({ budget_id: "" }); + const createBudget = useCreateBudget(); + + const handleCancel = () => { + setIsModalVisible(false); + form.reset(); + }; + + const handleCreate = async (formValues: BudgetFormValues) => { + try { + toast.info("Making API Call"); + await createBudget.mutateAsync(applyBudgetPrecision(formValues)); + toast.success("Budget Created"); + form.reset(); + setIsModalVisible(false); + } catch (error) { + console.error("Error creating the budget:", error); + toast.fromError(`Error creating the budget: ${error}`); + } + }; + + return ( + !open && handleCancel()}> + + + Create budget + +
+ + + + }>Cancel + + +
+
+ ); +}; + +export default BudgetModal; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.test.tsx similarity index 99% rename from ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.test.tsx index f2608c1221f..334bad8d0d8 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.test.tsx @@ -6,7 +6,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { ApiError } from "@/lib/http/client"; -import BudgetPanel from "./budget_panel"; +import BudgetPanel from "./BudgetPanel"; const { getMock, budgetDeleteMock } = vi.hoisted(() => ({ getMock: vi.fn(), diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.tsx similarity index 98% rename from ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx rename to ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.tsx index 630d90e91b9..785b09252af 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_panel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetPanel.tsx @@ -17,9 +17,9 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; import { toast } from "@/lib/toast"; import { useBudgetList, useDeleteBudget, budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; -import BudgetModal from "./budget_modal"; +import BudgetModal from "./BudgetModal"; import BudgetTable from "./BudgetTable"; -import EditBudgetModal from "./edit_budget_modal"; +import EditBudgetModal from "./EditBudgetModal"; import { CREATE_END_USER_CURL_COMMAND, CHAT_COMPLETIONS_CURL_COMMAND, OPENAI_SDK_PYTHON_CODE } from "./constants"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; import { isProxyAdminRole } from "@/utils/roles"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.integration.test.tsx new file mode 100644 index 00000000000..ea84d57ac30 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.integration.test.tsx @@ -0,0 +1,111 @@ +import { fireEvent, render, screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import React from "react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import type { components } from "@/lib/http/schema"; + +import EditBudgetModal from "./EditBudgetModal"; +import { chooseSelectOption } from "../../../../../tests/test-utils"; + +const { updateMock } = vi.hoisted(() => ({ updateMock: vi.fn() })); + +vi.mock("@/app/(dashboard)/hooks/budgets/useBudgets", () => ({ + useUpdateBudget: () => ({ mutateAsync: updateMock }), +})); + +type BudgetItem = components["schemas"]["BudgetListItem"]; + +const EXISTING_BUDGET: BudgetItem = { + budget_id: "budget-alpha", + max_budget: 12.345, + budget_duration: "7d", + tpm_limit: 1000, + rpm_limit: 10, + tpd_limit: 50000, + soft_budget: 25, + budget_reset_at: "2026-02-01T00:00:00Z", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-02T00:00:00Z", +}; + +const renderModal = () => + render(); + +const save = async (user: ReturnType) => + user.click(screen.getByRole("button", { name: "Save" })); + +const resetPeriod = () => screen.getByRole("combobox", { name: "Reset period" }); + +describe("EditBudgetModal", () => { + beforeEach(() => { + vi.clearAllMocks(); + updateMock.mockResolvedValue(undefined); + }); + + it("prefills every field from the existing budget", () => { + renderModal(); + + expect(screen.getByLabelText(/^Spend limit/)).toHaveValue(12.345); + expect(resetPeriod()).toHaveTextContent("per week"); + expect(screen.getByLabelText("Tokens per minute")).toHaveValue(1000); + expect(screen.getByLabelText("Requests per minute")).toHaveValue(10); + expect(screen.getByLabelText("Batch tokens per day")).toHaveValue(50000); + }); + + it("sends only the changed rate limit so the stored spend limit and reset period are left as they are", async () => { + const user = userEvent.setup(); + renderModal(); + + fireEvent.change(screen.getByLabelText("Tokens per minute"), { target: { value: "500" } }); + await save(user); + + await waitFor(() => expect(updateMock).toHaveBeenCalledTimes(1)); + expect(updateMock.mock.calls[0][0]).toEqual({ budget_id: "budget-alpha", tpm_limit: 500 }); + }); + + it("sends a later change after another field was changed and reverted", async () => { + const user = userEvent.setup(); + renderModal(); + + fireEvent.change(screen.getByLabelText("Tokens per minute"), { target: { value: "500" } }); + fireEvent.change(screen.getByLabelText("Tokens per minute"), { target: { value: "1000" } }); + fireEvent.change(screen.getByLabelText("Requests per minute"), { target: { value: "20" } }); + await save(user); + + await waitFor(() => expect(updateMock).toHaveBeenCalledTimes(1)); + expect(updateMock.mock.calls[0][0]).toEqual({ budget_id: "budget-alpha", rpm_limit: 20 }); + }); + + it("submits a changed spend limit and reset period", async () => { + const user = userEvent.setup(); + renderModal(); + + fireEvent.change(screen.getByLabelText(/^Spend limit/), { target: { value: "42.567" } }); + await chooseSelectOption(user, resetPeriod(), "per month"); + await save(user); + + await waitFor(() => expect(updateMock).toHaveBeenCalledTimes(1)); + const expected = { budget_id: "budget-alpha", max_budget: 42.57, budget_duration: "30d" }; + expect(updateMock.mock.calls[0][0]).toEqual(expected); + }); + + it("submits a cleared limit and an 'in total' reset period as null", async () => { + const user = userEvent.setup(); + renderModal(); + + await user.clear(screen.getByLabelText("Tokens per minute")); + await chooseSelectOption(user, resetPeriod(), "in total"); + await save(user); + + await waitFor(() => expect(updateMock).toHaveBeenCalledTimes(1)); + const expected = { budget_id: "budget-alpha", tpm_limit: null, budget_duration: null }; + expect(updateMock.mock.calls[0][0]).toEqual(expected); + }); + + it("does not let the budget ID be edited", () => { + renderModal(); + + expect(screen.getByLabelText("Budget ID")).toBeDisabled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.tsx new file mode 100644 index 00000000000..4aa69c5c15e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/EditBudgetModal.tsx @@ -0,0 +1,74 @@ +import React, { useEffect, useId } from "react"; +import { useUpdateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; +import { applyBudgetPrecision } from "./budgetPrecision"; +import { BudgetFormFields, type BudgetFormValues, useBudgetForm } from "./BudgetFormFields"; +import { pickDirty } from "@/lib/forms/pickDirty"; +import { toast } from "@/lib/toast"; +import { Button } from "@/components/ui/button"; +import { Dialog, DialogClose, DialogContent, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog"; + +const toFormValues = (budget: budgetItem) => ({ + budget_id: budget.budget_id, + tpm_limit: budget.tpm_limit, + rpm_limit: budget.rpm_limit, + tpd_limit: budget.tpd_limit, + max_budget: budget.max_budget, + budget_duration: budget.budget_duration, +}); + +interface EditBudgetModalProps { + isModalVisible: boolean; + setIsModalVisible: React.Dispatch>; + existingBudget: budgetItem; +} +const EditBudgetModal: React.FC = ({ isModalVisible, setIsModalVisible, existingBudget }) => { + const formId = useId(); + const form = useBudgetForm(toFormValues(existingBudget)); + const { dirtyFields } = form.formState; + const updateBudget = useUpdateBudget(); + + useEffect(() => { + form.reset(toFormValues(existingBudget)); + }, [existingBudget, form]); + + const handleCancel = () => { + setIsModalVisible(false); + form.reset(); + }; + + const handleUpdate = async (formValues: BudgetFormValues) => { + try { + toast.info("Making API Call"); + const changedValues = pickDirty(formValues, dirtyFields); + await updateBudget.mutateAsync(applyBudgetPrecision({ budget_id: formValues.budget_id, ...changedValues })); + toast.success("Budget Updated"); + form.reset(); + setIsModalVisible(false); + } catch (error) { + console.error("Error updating the budget:", error); + toast.fromError(`Error updating the budget: ${error}`); + } + }; + + return ( + !open && handleCancel()}> + + + Edit budget + +
+ + + + }>Cancel + + +
+
+ ); +}; + +export default EditBudgetModal; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.integration.test.tsx deleted file mode 100644 index 17a297d203d..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.integration.test.tsx +++ /dev/null @@ -1,138 +0,0 @@ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import React from "react"; -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import BudgetModal from "./budget_modal"; -import { chooseSelectOption } from "../../../../../tests/test-utils"; - -const { createMock } = vi.hoisted(() => ({ createMock: vi.fn() })); - -vi.mock("@/app/(dashboard)/hooks/budgets/useBudgets", () => ({ - useCreateBudget: () => ({ mutateAsync: createMock }), -})); - -const FULL_PAYLOAD = { - budget_id: "budget-alpha", - tpm_limit: 500.57, - rpm_limit: 7, - max_budget: 42.57, - budget_duration: "30d", -}; - -const renderModal = () => render(); - -const create = async (user: ReturnType) => - user.click(screen.getByRole("button", { name: "Create Budget" })); - -const openOptionalSettings = async (user: ReturnType) => { - await user.click(screen.getByText("Optional Settings")); - await screen.findByLabelText("Max Budget (USD)"); -}; - -describe("BudgetModal", () => { - beforeEach(() => { - vi.clearAllMocks(); - createMock.mockResolvedValue(undefined); - }); - - it("submits only the mounted fields when Optional Settings stays collapsed", async () => { - const user = userEvent.setup(); - renderModal(); - - fireEvent.change(screen.getByLabelText("Budget ID"), { target: { value: "budget-alpha" } }); - fireEvent.change(screen.getByLabelText("Max Tokens per minute"), { target: { value: "500.567" } }); - fireEvent.change(screen.getByLabelText("Max Requests per minute"), { target: { value: "7" } }); - await create(user); - - await waitFor(() => expect(createMock).toHaveBeenCalledTimes(1)); - expect(createMock.mock.calls[0][0]).toEqual({ - budget_id: "budget-alpha", - tpm_limit: 500.57, - rpm_limit: 7, - }); - }); - - it("submits every field once Optional Settings is expanded", async () => { - const user = userEvent.setup(); - renderModal(); - - fireEvent.change(screen.getByLabelText("Budget ID"), { target: { value: "budget-alpha" } }); - fireEvent.change(screen.getByLabelText("Max Tokens per minute"), { target: { value: "500.567" } }); - fireEvent.change(screen.getByLabelText("Max Requests per minute"), { target: { value: "7" } }); - - await openOptionalSettings(user); - fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "42.567" } }); - - await chooseSelectOption(user, screen.getByRole("combobox"), "monthly"); - - await create(user); - - await waitFor(() => expect(createMock).toHaveBeenCalledTimes(1)); - expect(createMock.mock.calls[0][0]).toEqual(FULL_PAYLOAD); - }); - - it("drops Optional Settings values again when the section is collapsed before submit", async () => { - const user = userEvent.setup(); - renderModal(); - - fireEvent.change(screen.getByLabelText("Budget ID"), { target: { value: "budget-alpha" } }); - - await openOptionalSettings(user); - fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "42.567" } }); - await chooseSelectOption(user, screen.getByRole("combobox"), "monthly"); - - await user.click(screen.getByText("Optional Settings")); - await waitFor(() => expect(screen.queryByLabelText("Max Budget (USD)")).not.toBeInTheDocument()); - await create(user); - - await waitFor(() => expect(createMock).toHaveBeenCalledTimes(1)); - expect(createMock.mock.calls[0][0]).toEqual({ budget_id: "budget-alpha" }); - }); - - it("submits a cleared number field as null", async () => { - const user = userEvent.setup(); - renderModal(); - - fireEvent.change(screen.getByLabelText("Budget ID"), { target: { value: "budget-alpha" } }); - fireEvent.change(screen.getByLabelText("Max Tokens per minute"), { target: { value: "5" } }); - await user.clear(screen.getByLabelText("Max Tokens per minute")); - await create(user); - - await waitFor(() => expect(createMock).toHaveBeenCalledTimes(1)); - expect(createMock.mock.calls[0][0]).toEqual({ - budget_id: "budget-alpha", - tpm_limit: null, - }); - }); - - it("blocks submit while Budget ID is empty", async () => { - const user = userEvent.setup(); - renderModal(); - - fireEvent.change(screen.getByLabelText("Max Tokens per minute"), { target: { value: "5" } }); - await create(user); - - await waitFor(() => expect(screen.getByLabelText("Budget ID")).toHaveAttribute("aria-invalid", "true")); - expect(createMock).not.toHaveBeenCalled(); - }); - - it("keeps a typed Optional Setting when the section is collapsed and reopened, as antd's store did", async () => { - const user = userEvent.setup(); - renderModal(); - fireEvent.change(screen.getByLabelText("Budget ID"), { target: { value: "probe-budget" } }); - - await openOptionalSettings(user); - fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "42.5" } }); - - await user.click(screen.getByText("Optional Settings")); - await user.click(screen.getByText("Optional Settings")); - - expect(await screen.findByLabelText("Max Budget (USD)")).toHaveValue(42.5); - - await create(user); - - await waitFor(() => expect(createMock).toHaveBeenCalledTimes(1)); - expect(createMock.mock.calls[0][0]).toMatchObject({ budget_id: "probe-budget", max_budget: 42.5 }); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx deleted file mode 100644 index 01f1595b365..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/budget_modal.tsx +++ /dev/null @@ -1,181 +0,0 @@ -import { ChevronRight } from "lucide-react"; -import React from "react"; -import { z } from "zod"; -import { useCreateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; -import { applyBudgetPrecision } from "./budgetPrecision"; -import { toast } from "@/lib/toast"; -import { FieldGroup } from "@/components/ui/field"; -import { FormField } from "@/components/shared/form/FormField"; -import { Button } from "@/components/ui/button"; -import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; -import { Input } from "@/components/ui/input"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { useZodForm } from "@/lib/forms/useZodForm"; -import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; - -const budgetShape = { - budget_id: z.string().min(1, "Please input a human-friendly name for the budget"), - tpm_limit: z.number().nullish(), - rpm_limit: z.number().nullish(), - tpd_limit: z.number().nullish(), - max_budget: z.number().nullish(), - budget_duration: z.string().nullish(), -}; - -const budgetSchema = z.object(budgetShape); - -type BudgetFormValues = z.output; - -const BUDGET_DURATION_OPTIONS = [ - { value: "24h", label: "daily" }, - { value: "7d", label: "weekly" }, - { value: "30d", label: "monthly" }, -]; - -interface BudgetModalProps { - isModalVisible: boolean; - setIsModalVisible: React.Dispatch>; -} -const BudgetModal: React.FC = ({ isModalVisible, setIsModalVisible }) => { - const [optionalSettingsOpen, setOptionalSettingsOpen] = React.useState(false); - const form = useZodForm(budgetSchema, { defaultValues: { budget_id: "" } }); - const createBudget = useCreateBudget(); - - const handleCancel = () => { - setIsModalVisible(false); - form.reset(); - }; - - const handleCreate = async (formValues: BudgetFormValues) => { - try { - toast.info("Making API Call"); - await createBudget.mutateAsync( - applyBudgetPrecision( - optionalSettingsOpen ? formValues : { ...formValues, max_budget: undefined, budget_duration: undefined }, - ), - ); - toast.success("Budget Created"); - form.reset(); - setIsModalVisible(false); - } catch (error) { - console.error("Error creating the budget:", error); - toast.fromError(`Error creating the budget: ${error}`); - } - }; - - return ( - !open && handleCancel()}> - - - Create Budget - -
- - - {({ ref, ...field }) => } - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - - - Optional Settings - - - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - - )} - - - - - -
- -
- -
-
- ); -}; - -export default BudgetModal; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.integration.test.tsx deleted file mode 100644 index fe0ecfc10dd..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.integration.test.tsx +++ /dev/null @@ -1,112 +0,0 @@ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import React from "react"; -import { beforeEach, describe, expect, it, vi } from "vitest"; - -import type { components } from "@/lib/http/schema"; - -import EditBudgetModal from "./edit_budget_modal"; -import { chooseSelectOption } from "../../../../../tests/test-utils"; - -const { updateMock } = vi.hoisted(() => ({ updateMock: vi.fn() })); - -vi.mock("@/app/(dashboard)/hooks/budgets/useBudgets", () => ({ - useUpdateBudget: () => ({ mutateAsync: updateMock }), -})); - -type BudgetItem = components["schemas"]["BudgetListItem"]; - -const EXISTING_BUDGET: BudgetItem = { - budget_id: "budget-alpha", - max_budget: 100, - budget_duration: "7d", - tpm_limit: 1000, - rpm_limit: 10, - soft_budget: 25, - budget_reset_at: "2026-02-01T00:00:00Z", - created_at: "2026-01-01T00:00:00Z", - updated_at: "2026-01-02T00:00:00Z", -}; - -const renderModal = () => - render(); - -const save = async (user: ReturnType) => - user.click(screen.getByRole("button", { name: "Save" })); - -const openOptionalSettings = async (user: ReturnType) => { - await user.click(screen.getByText("Optional Settings")); - await screen.findByLabelText("Max Budget (USD)"); -}; - -describe("EditBudgetModal", () => { - beforeEach(() => { - vi.clearAllMocks(); - updateMock.mockResolvedValue(undefined); - }); - - it("submits only the mounted fields when Optional Settings stays collapsed", async () => { - const user = userEvent.setup(); - renderModal(); - - await user.clear(screen.getByLabelText("Max Tokens per minute")); - fireEvent.change(screen.getByLabelText("Max Tokens per minute"), { target: { value: "500.567" } }); - await save(user); - - await waitFor(() => expect(updateMock).toHaveBeenCalledTimes(1)); - expect(updateMock.mock.calls[0][0]).toEqual({ - budget_id: "budget-alpha", - tpm_limit: 500.57, - rpm_limit: 10, - }); - }); - - it("submits every field once Optional Settings is expanded", async () => { - const user = userEvent.setup(); - renderModal(); - - await user.clear(screen.getByLabelText("Max Tokens per minute")); - fireEvent.change(screen.getByLabelText("Max Tokens per minute"), { target: { value: "500.567" } }); - await user.clear(screen.getByLabelText("Max Requests per minute")); - fireEvent.change(screen.getByLabelText("Max Requests per minute"), { target: { value: "7" } }); - - await openOptionalSettings(user); - await user.clear(screen.getByLabelText("Max Budget (USD)")); - fireEvent.change(screen.getByLabelText("Max Budget (USD)"), { target: { value: "42.567" } }); - - await chooseSelectOption(user, screen.getByRole("combobox"), "monthly"); - - await save(user); - - await waitFor(() => expect(updateMock).toHaveBeenCalledTimes(1)); - const expected = { - budget_id: "budget-alpha", - tpm_limit: 500.57, - rpm_limit: 7, - max_budget: 42.57, - budget_duration: "30d", - }; - - expect(updateMock.mock.calls[0][0]).toEqual(expected); - }); - - it("keeps a typed Optional Setting when the section is collapsed and reopened, as antd's store did", async () => { - const user = userEvent.setup(); - renderModal(); - - await openOptionalSettings(user); - const maxBudget = screen.getByLabelText("Max Budget (USD)"); - await user.clear(maxBudget); - fireEvent.change(maxBudget, { target: { value: "99.25" } }); - - await user.click(screen.getByText("Optional Settings")); - await user.click(screen.getByText("Optional Settings")); - - expect(await screen.findByLabelText("Max Budget (USD)")).toHaveValue(99.25); - - await save(user); - - await waitFor(() => expect(updateMock).toHaveBeenCalledTimes(1)); - expect(updateMock.mock.calls[0][0]).toMatchObject({ max_budget: 99.25 }); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx deleted file mode 100644 index 1931a88f096..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/edit_budget_modal.tsx +++ /dev/null @@ -1,187 +0,0 @@ -import { ChevronRight } from "lucide-react"; -import React, { useEffect } from "react"; -import { useForm } from "react-hook-form"; -import { useUpdateBudget } from "@/app/(dashboard)/hooks/budgets/useBudgets"; -import { budgetItem } from "@/app/(dashboard)/hooks/budgets/useBudgets"; -import { applyBudgetPrecision } from "./budgetPrecision"; -import { toast } from "@/lib/toast"; -import { FieldGroup } from "@/components/ui/field"; -import { FormField } from "@/components/shared/form/FormField"; -import { Button } from "@/components/ui/button"; -import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"; -import { Input } from "@/components/ui/input"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/ui/dialog"; - -type EditBudgetFormValues = Pick< - budgetItem, - "budget_id" | "tpm_limit" | "rpm_limit" | "tpd_limit" | "max_budget" | "budget_duration" ->; - -const toFormValues = (budget: budgetItem): EditBudgetFormValues => ({ - budget_id: budget.budget_id, - tpm_limit: budget.tpm_limit, - rpm_limit: budget.rpm_limit, - tpd_limit: budget.tpd_limit, - max_budget: budget.max_budget, - budget_duration: budget.budget_duration, -}); - -const BUDGET_DURATION_OPTIONS = [ - { value: "24h", label: "daily" }, - { value: "7d", label: "weekly" }, - { value: "30d", label: "monthly" }, -]; - -interface EditBudgetModalProps { - isModalVisible: boolean; - setIsModalVisible: React.Dispatch>; - existingBudget: budgetItem; -} -const EditBudgetModal: React.FC = ({ isModalVisible, setIsModalVisible, existingBudget }) => { - const [optionalSettingsOpen, setOptionalSettingsOpen] = React.useState(false); - const form = useForm({ defaultValues: toFormValues(existingBudget) }); - const updateBudget = useUpdateBudget(); - - useEffect(() => { - form.reset(toFormValues(existingBudget)); - }, [existingBudget, form]); - - const handleCancel = () => { - setIsModalVisible(false); - form.reset(); - }; - - const handleUpdate = async (formValues: EditBudgetFormValues) => { - try { - toast.info("Making API Call"); - await updateBudget.mutateAsync( - applyBudgetPrecision( - optionalSettingsOpen ? formValues : { ...formValues, max_budget: undefined, budget_duration: undefined }, - ), - ); - toast.success("Budget Updated"); - form.reset(); - setIsModalVisible(false); - } catch (error) { - console.error("Error updating the budget:", error); - toast.fromError(`Error updating the budget: ${error}`); - } - }; - - return ( - !open && handleCancel()}> - - - Edit Budget - -
- - - {({ ref, ...field }) => } - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - - - Optional Settings - - - - - {({ ref, value, onChange, ...field }) => ( - onChange(event.target.value === "" ? null : event.target.valueAsNumber)} - /> - )} - - - {({ id, value, onChange, "aria-invalid": ariaInvalid, "aria-describedby": ariaDescribedBy }) => ( - - )} - - - - - -
- -
- -
-
- ); -}; - -export default EditBudgetModal; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/page.tsx index ca34589a679..7cac0788f73 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/page.tsx @@ -1,6 +1,6 @@ "use client"; -import BudgetPanel from "./_components/budget_panel"; +import BudgetPanel from "./_components/BudgetPanel"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function Budgets() { From e333ccabdbbfdeff138fc348c29b6e7cbc692a1c Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 00:59:23 +0000 Subject: [PATCH 035/235] fix(scim)!: delete a group through the team delete's bulk path instead of one member at a time (#47544) * fix(scim): write a group's roster in one locked transaction instead of one member at a time POST, PUT and PATCH /scim/v2/Groups resolve every member in one query and write the roster, the membership rows and the users' team tags in one transaction under the team's advisory lock, so a 500-member push runs a fixed number of statements instead of thousands. PUT now answers with the roster it just wrote * test(scim): assert the roster the IdP gets back and boot the bulk roster cell with two workers * fix(scim): skip the team lock for a rename-only PATCH and lock member rows in id order before the roster write A PATCH naming no members opened the roster transaction and took the team advisory lock for nothing. A SIGKILLed worker then left its orphaned query engine holding that lock until the 60 s transaction timeout, and /team/delete, which takes the same lock, timed out behind it. An empty delta is now answered from a plain team read, with no transaction and no lock Two groups trading members in parallel locked user rows in opposite orders, and one request died with a Postgres deadlock. Every user row a push touches is now locked with SELECT ... ORDER BY user_id FOR UPDATE before the first write, so the two requests serialize The bulk roster cell gets a 120 s client through a new owned_proxy_process client_timeout, so an unfixed push fails on the statement count instead of the harness clock, requires positive counts on both pushes, and gains a parallel roster-trade test that answered 500 before the lock * fix(scim): evict the cached team on a group rename and read a no-op roster from the writer A rename-only PUT or PATCH wrote the new alias and left the team cached under its id and old alias on every worker, so keys kept logging spend against the old team alias until the TTL expired. The group write now evicts the cached team (and broadcasts the eviction) right after the alias or metadata update. sync_team_roster reads the team from the writer when the plan names nobody, so a replica cannot hand back the pre-rename row; a plan already matching the roster returns inside the lock without writing, evicting, or emitting the members metric; the response is built from the written row, so lastModified reflects the roster write; and detaching a removed member bumps the user's updated_at like every other user write. * fix(scim): bump detached members' updated_at in UTC The detach UPDATE wrote updated_at = CURRENT_TIMESTAMP, which Postgres casts into Prisma's naive timestamp column in the session timezone, so on a database whose session is not UTC the removed member's row landed hours away from the UTC instants Prisma itself writes. Write NOW() AT TIME ZONE 'UTC' instead, and have the bulk roster cell assert the bumped row sits between two reads of the database's own UTC clock taken around the replace. * fix(scim): delete a group through the team delete's bulk path instead of one member at a time * fix(scim): chunk the group write's member lookups under the bind cap Every Prisma "in" filter the group write issues bound one parameter per member with nothing bounding the list, which check_unbounded_in_lists reports as 11 new findings. The single-field lookups now go through litellm.repositories.chunked_in (caller_transaction inside the roster transaction, per_chunk_ok for the role recompute), and the one cross-field OR read slices its values at IN_LIST_CHUNK_SIZE and records the bound. A push of IN_LIST_CHUNK_SIZE + 1 members now reads in two slices, and a member named only in the last slice still resolves. * fix(scim): demote a deleted admin group's members before the team delete so a failed delete stays retryable * fix(scim): recompute member roles again after the team delete so a concurrent group write cannot regrant admin * fix(proxy): send bulk cache evictions in slices so the broadcaster drops none Veria pointed out that the invalidation publisher caps its pending tasks at 1,024 and drops the excess with a warning, and that a group push now detaches every removed member's keys in one call instead of one member at a time, so a removal of more than roughly 340 members (three cache keys each) silently left stale key objects on the other replicas. The merge base never hit the cap because each member was a separate request. Bulk evictions (team, user, organization, and session key deletes, plus the roster settle) now publish in slices of 256 and wait, bounded at two seconds, for the backlog already handed to the event loop before each slice. A Redis that never answers costs a bulk caller at most that wait per slice instead of wedging the request, and single-key callers are untouched. Regression tests push 1,500 keys and a 500-member removal through a slow publisher and assert every eviction reaches Redis; both fail against the previous tip. * fix(proxy): clear this worker's cache before pacing bulk eviction broadcasts Greptile pointed out that pacing the broadcasts in slices also paced the local eviction: a bulk delete waited for the publish backlog before dropping the next slice from this worker's own memory, so on a Redis that accepts connections but never answers a key deleted from the database kept authenticating on the handling worker for up to two seconds per slice. The merge base cleared every key locally at once. Bulk evictions now drop every affected entry from this worker's memory first, through the cache's own partition lookup since key objects live apart from the other auth objects, and only then publish in paced slices. The roster settle does the same for the deleted tokens, the JWT mapping keys and every changed member's user and membership entries before any step that can wait. Regression tests run both paths against a publisher that never answers and check that nothing is still cached on this worker before the first bound elapses; both fail with the local eviction removed. * fix(scim): write recomputed member roles only where the membership still backs them * fix(team): report the users a team delete detaches so SCIM demotes late-added members * test(scim): audit cells for the bulk group delete Move the SCIM group helpers into tests/integration/_support/scim.py and add the delete cells: a member's other group survives, the admin group's members are demoted and the audit row names the SCIM caller, a member a concurrent PUT adds while the delete waits on the team lock is demoted too, unknown, repeated, non-admin and hostile ids, an empty group, the peer proxy, a 20-delete burst across both proxies, and a delete after a worker was SIGKILLed * fix(proxy): drop the Redis copies of bulk-evicted auth objects before pacing the broadcasts * docs(proxy): drop the module docstring that repeated the PR description * fix(team): sweep deleted team references only under the lock so a late-added member is demoted The unlocked sweep that ran before the team's advisory lock stripped a member a concurrent group write added after the roster was read, so neither the roster snapshot nor the locked sweep's RETURNING named them and the SCIM recompute left them proxy_admin after the admin group was gone. The locked sweep does the same strip and reports whom it stripped, so it is the only sweep now. The integration cell holds the delete on a member's user row so a PATCH lands between the roster read and the lock, then checks the late member is demoted. Both waiter queries in the SCIM support module return their count as text so the first poll parses. * chore(team): carry the unbounded-IN baseline entries to delete_validated_teams The delete pass moved from delete_team and the unlocked sweep into delete_validated_teams, so the checker's baseline names the function the two team_id IN lists now live in * test(scim): expect the master key alias in the audit row and send post-kill requests on a fresh connection * fix(scim): bound bulk cache evictions to one Redis round trip and close lock-order gaps Bulk key evictions now drop entries from memory, delete them from Redis in one chunked DEL, and broadcast every key in one pipelined publish, so a large roster removal no longer overflows the publisher's pending cap or costs a Redis round trip per member. A push waits for in-flight membership loads before evicting. User rows are locked in byte order to match the spend writer, a bulk user delete takes every team lock before its first user write, and a POST whose roster write fails deletes the empty team so the IdP retry creates it again. * test(team): observe delete broadcasts at the Redis layer * fix(scim): lock user rows in id order on every multi-row writer and create unprovisioned members concurrently * fix(team): chunk the email lookup behind the member row lock * fix(scim): create a member deleted mid-push again instead of failing the group write * fix(scim): refuse strict-mode mid-push deletes with 400 and check license seats once per batch A member deleted mid-push is created again only when scim_upsert_user allows it, under the verb's own created_via. New group members are checked against the license seat limit as one batch before any is created. Auth cache publish drops log the key count instead of the key. * fix(scim): lock member rows while recomputing admin roles after a group push * fix(users): re-read teams under the row locks in bulk user delete so a racing group push keeps no deleted member * fix(users): take /user/delete and SCIM DELETE /Users users off their current teams under the row locks so a racing group push keeps no deleted member * fix(scim): keep retrying a push while deletes race it, lock rows in member bulk delete, evict removed keys from Redis first A group push gave up after one retry when SCIM user deletes kept landing between attempts, answering 500 in 9 of 10 race rounds. It now retries while each attempt finds newly deleted members, and gives up only when a member it created again goes missing a second time. The team member bulk delete now locks the removed members' user rows in the shared order and takes the team off their teams arrays in one array_remove statement, so it no longer writes back a teams array read before a concurrent push added another team. A push that removes members now drops their deleted team keys from memory and Redis together with the members' other cache entries before the team eviction, so a removed key cannot refill from Redis mid-cleanup. * fix(user-delete): keep the team alias label on the team members gauge after removals * test(integration): count relay statements over plaintext so an SSL-enabled Postgres does not hide them * fix(user-delete): skip a team deleted while the user delete waited on its lock instead of answering 404 * fix(scim): answer 404 and roll back when the team is deleted during a roster write * test(integration): audit cells for SCIM bulk roster writes, user delete locking and the eviction broadcast * test(scim): cover deleting a SCIM group through the team delete's bulk path * fix(scim): evict a deleted group's member caches in bulk and drop cached roles after a recompute * fix(scim): answer 404 to a group delete that lost a race to another delete * fix(scim): chunk the locked team lookup and delete in the team delete transaction * fix(scim): keep service-account keys and team models on a group delete and write its records once under the lock * fix(scim): demote a deleted group's members inside the delete transaction The admin-group role write now runs on the team delete's transaction, over the roster read under the team lock, so a failed delete leaves roles untouched and a member added while the delete waited is demoted with the rest. The delete transaction also waits for a free connection as long as it may run, so a burst of deletes queues instead of failing after Prisma's two-second default. Every member in the 5-vs-500 statement test now holds a key. * fix(audit): write deleted keys' audit rows in one insert A delete that removes many keys wrote one audit insert task per key, and those tasks held pool connections the delete's follow-up writes waited on. The race test now waits on any lock wait behind the delete. --- .../proxy/hooks/key_management_event_hooks.py | 56 ++- .../management_endpoints/scim/scim_v2.py | 116 +++-- .../management_endpoints/team_endpoints.py | 389 +++++++++++----- .../proxy/management_helpers/audit_logs.py | 61 ++- .../management_helpers/team_roster_sync.py | 14 +- litellm/proxy/utils.py | 9 +- .../unbounded_in_baseline.txt | 4 +- tests/integration/_support/scim.py | 207 +++++++++ .../management/test_scim_group_bulk_roster.py | 272 +++++------ .../management/test_scim_group_delete.py | 431 ++++++++++++++++++ .../hooks/test_key_management_event_hooks.py | 40 +- .../scim/test_scim_v2_endpoints.py | 194 ++++++-- .../test_internal_user_endpoints.py | 6 +- .../test_team_endpoints.py | 391 ++++++++++++++-- .../test_bulk_user_deletion.py | 14 +- 15 files changed, 1744 insertions(+), 460 deletions(-) create mode 100644 tests/integration/_support/scim.py create mode 100644 tests/integration/management/test_scim_group_delete.py diff --git a/litellm/proxy/hooks/key_management_event_hooks.py b/litellm/proxy/hooks/key_management_event_hooks.py index ef60a800cdd..47c3869fb64 100644 --- a/litellm/proxy/hooks/key_management_event_hooks.py +++ b/litellm/proxy/hooks/key_management_event_hooks.py @@ -254,39 +254,45 @@ class KeyManagementEventHooks: litellm_changed_by: str | None = None, ) -> None: from litellm.proxy.management_helpers.audit_logs import ( - create_audit_log_for_update, + create_audit_logs_for_updates, get_audit_log_changed_by, is_audit_logging_enabled, + track_audit_task, ) from litellm.proxy.proxy_server import litellm_proxy_admin_name if not is_audit_logging_enabled(): return - for key in keys_being_deleted: - key_row = LiteLLM_VerificationToken.model_validate(key, from_attributes=True) - if key_row.token is None: - continue - _key_row = key_row.model_dump_json(exclude_none=True) - - asyncio.create_task( - create_audit_log_for_update( - request_data=LiteLLM_AuditLogs( - id=str(uuid.uuid4()), - updated_at=datetime.now(timezone.utc), - changed_by=get_audit_log_changed_by( - litellm_changed_by=litellm_changed_by, - user_api_key_dict=user_api_key_dict, - litellm_proxy_admin_name=litellm_proxy_admin_name, - ), - changed_by_api_key=user_api_key_dict.token, - table_name=LitellmTableNames.KEY_TABLE_NAME, - object_id=key_row.token, - action="deleted", - updated_values="{}", - before_value=_key_row, - ) - ) + changed_by: Final = get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ) + key_rows: Final = ( + LiteLLM_VerificationToken.model_validate(key, from_attributes=True) for key in keys_being_deleted + ) + audit_rows: Final = tuple( + LiteLLM_AuditLogs( + id=str(uuid.uuid4()), + updated_at=datetime.now(timezone.utc), + changed_by=changed_by, + changed_by_api_key=user_api_key_dict.token, + table_name=LitellmTableNames.KEY_TABLE_NAME, + object_id=key_row.token, + action="deleted", + updated_values="{}", + before_value=key_row.model_dump_json(exclude_none=True), ) + for key_row in key_rows + if key_row.token is not None + ) + if not audit_rows: + return + track_audit_task( + asyncio.create_task( + create_audit_logs_for_updates(audit_rows), name=f"{len(audit_rows)} deleted key audit logs" + ) + ) @staticmethod async def _store_virtual_key_in_secret_manager(secret_name: str, secret_token: str, team_id: str | None = None): diff --git a/litellm/proxy/management_endpoints/scim/scim_v2.py b/litellm/proxy/management_endpoints/scim/scim_v2.py index ec905a9f187..668bd64227b 100644 --- a/litellm/proxy/management_endpoints/scim/scim_v2.py +++ b/litellm/proxy/management_endpoints/scim/scim_v2.py @@ -11,7 +11,7 @@ from dataclasses import dataclass from functools import partial, reduce from itertools import chain, groupby from types import MappingProxyType -from typing import TYPE_CHECKING, Final, NamedTuple, NoReturn, Protocol, TypeAlias, overload +from typing import TYPE_CHECKING, Annotated, Final, NamedTuple, NoReturn, Protocol, TypeAlias, overload from fastapi import ( APIRouter, @@ -53,7 +53,12 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo delete_cache_team_object, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( + broadcast_evictions, + evict_and_broadcast, + evict_local, + evict_shared, +) from litellm.proxy.common_utils.http_parsing_utils import ( # noqa: F401 # legacy module exports _safe_get_request_headers, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export safe_get_request_headers, @@ -64,6 +69,7 @@ from litellm.proxy.management_endpoints.scim.scim_transformations import ( ScimTransformations, ) from litellm.proxy.management_endpoints.team_endpoints import ( + delete_validated_team_rows, new_team, team_member_add, team_member_delete, @@ -104,6 +110,7 @@ from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.proxy.management_endpoints.scim_v2 import * if TYPE_CHECKING: + from prisma import Prisma from prisma.models import LiteLLM_VerificationToken as PrismaVerificationToken @@ -317,11 +324,15 @@ async def _check_team_exists(team_id: str) -> LiteLLM_TeamTable: team: Final = await _table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id}) if not team: - raise HTTPException(status_code=404, detail={"error": f"Group not found with ID: {team_id}"}) + raise _group_not_found(team_id) return team +def _group_not_found(team_id: str) -> HTTPException: + return HTTPException(status_code=404, detail={"error": f"Group not found with ID: {team_id}"}) + + def _extract_scim_user_data(user: SCIMUser) -> ScimUserData: """Extract common data from SCIMUser object.""" user_email = None @@ -482,10 +493,7 @@ async def _recompute_scim_member_roles(prisma_client: PrismaClient, user_ids: It Recompute and persist each user's global proxy role from their resulting team membership. No-op unless scim_admin_group is configured, so a SCIM group write that drops a member from the admin group demotes them just like the user - endpoints do, and the role is left untouched when the feature is off. The users' - rows are locked through ``USER_ROWS_LOCK_SQL`` before their teams are read, the - same lock a roster write takes, so a concurrent group push cannot change a - member's teams between the read and the role write. + endpoints do, and the role is left untouched when the feature is off. """ admin_group: Final = await _get_scim_admin_group() if admin_group is None: @@ -494,28 +502,46 @@ async def _recompute_scim_member_roles(prisma_client: PrismaClient, user_ids: It ids: Final = tuple(dict.fromkeys(user_ids)) if not ids: return - default_role: Final = _default_scim_user_role() async with prisma_client.tx(timeout=ROSTER_TX_TIMEOUT) as tx: - await tx.query_raw(USER_ROWS_LOCK_SQL, sorted(ids)) - rows: Final = await find_many_in(user_tx_db(tx), "user_id", ids) - team_ids: Final = tuple(dict.fromkeys(chain.from_iterable(row.teams for row in rows))) - teams: Final = await find_many_in(team_tx_db(tx), "team_id", team_ids) - alias_of: Final = MappingProxyType({team.team_id: team.team_alias for team in teams}) - resolved: Final = tuple( - ( - row.user_id, - _resolve_scim_user_role( - [SCIMUserGroup(value=team_id, display=alias_of.get(team_id)) for team_id in row.teams], - admin_group, - default_role, - ), - ) - for row in rows + await _write_scim_member_roles_tx(tx, ids, admin_group=admin_group) + + from litellm.proxy.proxy_server import user_api_key_cache + + evict_local(cache_keys=ids, user_api_key_cache=user_api_key_cache) + await evict_shared(cache_keys=ids, cache=user_api_key_cache) + await broadcast_evictions(cache_keys=ids) + + +async def _write_scim_member_roles_tx(tx: "Prisma", user_ids: Sequence[str], *, admin_group: str) -> None: + """ + Write each user's role resolved from the teams they are in, on ``tx``. The users' + rows are locked through ``USER_ROWS_LOCK_SQL`` before their teams are read, the + same lock a roster write takes, so a concurrent group push cannot change a + member's teams between the read and the role write. + """ + if not user_ids: + return + default_role: Final = _default_scim_user_role() + await tx.query_raw(USER_ROWS_LOCK_SQL, sorted(user_ids)) + rows: Final = await find_many_in(user_tx_db(tx), "user_id", user_ids) + team_ids: Final = tuple(dict.fromkeys(chain.from_iterable(row.teams for row in rows))) + teams: Final = await find_many_in(team_tx_db(tx), "team_id", team_ids) + alias_of: Final = MappingProxyType({team.team_id: team.team_alias for team in teams}) + resolved: Final = tuple( + ( + row.user_id, + _resolve_scim_user_role( + [SCIMUserGroup(value=team_id, display=alias_of.get(team_id)) for team_id in row.teams], + admin_group, + default_role, + ), + ) + for row in rows + ) + for role in dict.fromkeys(role for _, role in resolved): + await tx.execute_raw( + _SET_USER_ROLE_SQL, role, sorted(user_id for user_id, user_role in resolved if user_role == role) ) - for role in dict.fromkeys(role for _, role in resolved): - await tx.execute_raw( - _SET_USER_ROLE_SQL, role, sorted(user_id for user_id, user_role in resolved if user_role == role) - ) class _ResolvedUserMember(NamedTuple): @@ -3051,7 +3077,8 @@ async def update_group( dependencies=[Depends(user_api_key_auth)], ) async def delete_group( - group_id: str = Path(..., title="Group ID"), + group_id: Annotated[str, Path(title="Group ID")], + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): """ Delete a group according to SCIM v2 protocol @@ -3064,23 +3091,22 @@ async def delete_group( existing_team: Final = await _check_team_exists(group_id) raise_if_teams_hold_ptu_shares((group_id,), llm_router) - member_ids: Final = await _get_team_member_user_ids_from_team(existing_team) - - # For each member, remove this team from their teams list - for member_id in member_ids: - user = await _table(UserRepository(prisma_client)).find_unique(where={"user_id": member_id}) - if user: - current_teams = user.teams or [] - if group_id in current_teams: - new_teams = [t for t in current_teams if t != group_id] - await _table(UserRepository(prisma_client)).update( - where={"user_id": member_id}, data={"teams": new_teams} - ) - - await _recompute_scim_member_roles(prisma_client, member_ids) - - # Delete team - await _table(TeamRepository(prisma_client)).delete(where={"team_id": group_id}) + admin_group: Final = await _get_scim_admin_group() + deleted: Final = await delete_validated_team_rows( + teams=(LiteLLM_TeamTable.model_validate(existing_team.model_dump()),), + prisma_client=prisma_client, + user_api_key_dict=UserAPIKeyAuth( + user_role=LitellmUserRoles.PROXY_ADMIN, + api_key=user_api_key_dict.api_key, + user_id=user_api_key_dict.user_id, + ), + litellm_changed_by=None, + before_commit=( + None if admin_group is None else partial(_write_scim_member_roles_tx, admin_group=admin_group) + ), + ) + if group_id not in deleted.team_ids: + raise _group_not_found(group_id) return Response(status_code=204) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 517a05773c8..d9763f72741 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -14,7 +14,7 @@ import copy import json import math import traceback -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass from datetime import datetime, timedelta, timezone @@ -34,7 +34,7 @@ from typing import ( import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status -from pydantic import BaseModel, Field, JsonValue, TypeAdapter, ValidationError +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict, assert_never import litellm @@ -102,6 +102,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo OrganizationNotFoundError, _cache_team_object, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export allowed_route_check_inside_route, + await_inflight_team_membership_load, cache_team_object, can_org_access_model, delete_cache_key_objects, @@ -112,6 +113,7 @@ from litellm.proxy.auth.auth_checks import ( # noqa: F401 # legacy module expo get_team_object, get_user_object, invalidate_team_member_spend_state, + jwt_key_mapping_cache_key, ) from litellm.proxy.auth.auth_utils import ( enforce_batch_limits_are_admin_only, @@ -119,12 +121,19 @@ from litellm.proxy.auth.auth_utils import ( ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( + broadcast_evictions, evict_and_broadcast, + evict_local, + evict_shared, publish_auth_cache_invalidation, ) from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch -from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache +from litellm.proxy.common_utils.user_api_key_cache import ( + UserApiKeyCache, + team_membership_auth_cache_key, + team_membership_reservation_cache_key, +) from litellm.proxy.db.db_span import db_span from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.hooks.model_max_budget_limiter import ( @@ -207,7 +216,7 @@ from litellm.proxy.management_helpers.utils import ( ) from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy from litellm.repositories.budget_repository import BudgetRepository -from litellm.repositories.chunked_in import find_many_in +from litellm.repositories.chunked_in import delete_many_in, find_many_in from litellm.repositories.organization_repository import OrganizationRepository from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import ( @@ -417,6 +426,23 @@ class _TeamDeleteTx(AccessGroupSyncTx, Protocol): @property def litellm_teammembership(self) -> "TableActions[prisma_models.LiteLLM_TeamMembership]": ... + @property + def litellm_deletedteamtable(self) -> "TableActions[prisma_models.LiteLLM_DeletedTeamTable]": ... + + @property + def litellm_verificationtoken(self) -> "TableActions[prisma_models.LiteLLM_VerificationToken]": ... + + @property + def litellm_jwtkeymapping(self) -> "TableActions[prisma_models.LiteLLM_JWTKeyMapping]": ... + + +def _team_delete_tx(tx: "Prisma") -> _TeamDeleteTx: + return tx # pyright: ignore[reportReturnType] # TableActions widens the generated inputs to Mapping, as the repositories do + + +_LOCK_DELETED_TEAM_MEMBERS_SQL: Final = """ +SELECT user_id, teams FROM "LiteLLM_UserTable" WHERE teams && $1::text[] ORDER BY user_id COLLATE "C" FOR UPDATE +""" _STRIP_DELETED_TEAMS_FROM_USERS_SQL: Final = """ UPDATE "LiteLLM_UserTable" @@ -430,6 +456,22 @@ WHERE user_id IN ( ) """ + +class _DetachedMember(LiteLLMBaseModel): + model_config = ConfigDict(frozen=True) + + user_id: str + teams: tuple[str, ...] + + +_DetachedMembers: Final = TypeAdapter(tuple[_DetachedMember, ...]) + + +@dataclass(frozen=True, slots=True) +class DeletedTeams: + team_ids: tuple[str, ...] + + _INCLUDE_MODEL_TABLE: Final = MappingProxyType({"litellm_model_table": True}) @@ -4526,18 +4568,7 @@ async def delete_team( }' ``` """ - from litellm.proxy.management_helpers.audit_logs import ( - get_audit_log_changed_by, - is_audit_logging_enabled, - ) - from litellm.proxy.proxy_server import ( - create_audit_log_for_update, - litellm_proxy_admin_name, - llm_router, - prisma_client, - proxy_logging_obj, - user_api_key_cache, - ) + from litellm.proxy.proxy_server import llm_router, prisma_client if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -4567,47 +4598,32 @@ async def delete_team( raise_if_teams_hold_ptu_shares(data.team_ids, llm_router) - await _persist_deleted_team_records( + await delete_validated_teams( teams=team_rows, prisma_client=prisma_client, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by, ) + deleted_teams: Final[_DeletedTeamsResult] = {"deleted_teams": data.team_ids} + return deleted_teams - # we do this after the first for loop, since first for loop is for validation. we only want this inserted after validation passes - if is_audit_logging_enabled(): - # make an audit log for each team deleted - for team_id in data.team_ids: - team_row: LiteLLM_TeamTable | None = await prisma_client.get_data( - team_id=team_id, table_name="team", query_type="find_unique" - ) - if team_row is None: - continue +async def delete_validated_teams( + teams: Sequence[LiteLLM_TeamTable], + prisma_client: PrismaClient, + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None, +) -> DeletedTeams: + """ + What ``/team/delete`` does once the teams exist and the caller may delete them: every key + and BYOK model of the teams, then ``delete_validated_team_rows``. + """ + from litellm.proxy.proxy_server import ( + proxy_logging_obj, + user_api_key_cache, + ) - _team_row = team_row.json(exclude_none=True) - - asyncio.create_task( - create_audit_log_for_update( - request_data=LiteLLM_AuditLogs( - id=str(uuid.uuid4()), - updated_at=datetime.now(timezone.utc), - changed_by=get_audit_log_changed_by( - litellm_changed_by=litellm_changed_by, - user_api_key_dict=user_api_key_dict, - litellm_proxy_admin_name=litellm_proxy_admin_name, - ), - changed_by_api_key=user_api_key_dict.api_key, - table_name=LitellmTableNames.TEAM_TABLE_NAME, - object_id=team_id, - action="deleted", - updated_values="{}", - before_value=_team_row, - ) - ) - ) - - # End of Audit logging + team_ids: Final = [team.team_id for team in teams] ## DELETE ASSOCIATED KEYS # Fetch keys before deletion to persist them @@ -4615,7 +4631,7 @@ async def delete_team( persist_deleted_verification_tokens, ) - keys_to_delete: Final = await _tokens_db(prisma_client).find_many(where={"team_id": {"in": data.team_ids}}) + keys_to_delete: Final = await _tokens_db(prisma_client).find_many(where={"team_id": {"in": team_ids}}) jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( hashed_tokens=tuple(key.token for key in keys_to_delete), prisma_client=prisma_client, @@ -4629,7 +4645,7 @@ async def delete_team( litellm_changed_by=litellm_changed_by, ) - await prisma_client.delete_data(team_id_list=data.team_ids, table_name="key") + await prisma_client.delete_data(team_id_list=team_ids, table_name="key") if keys_to_delete: KeyManagementEventHooks.create_key_deleted_audit_logs( @@ -4654,18 +4670,41 @@ async def delete_team( from litellm.proxy.proxy_server import llm_router await delete_team_models( - team_ids=data.team_ids, + team_ids=team_ids, prisma_client=prisma_client, llm_router=llm_router, ) - await _sweep_deleted_team_references(team_ids=data.team_ids, prisma_client=prisma_client) - - member_ids_per_team: Final = await _resolve_deleted_team_member_user_ids( - teams=team_rows, + return await delete_validated_team_rows( + teams=teams, prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, ) + +async def delete_validated_team_rows( + teams: Sequence[LiteLLM_TeamTable], + prisma_client: PrismaClient, + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None, + before_commit: Callable[["Prisma", tuple[str, ...]], Awaitable[None]] | None = None, +) -> DeletedTeams: + """ + Under every team's advisory lock, delete the teams still there with the keys their rostered + members hold on them, their membership rows and user references, and write their deleted-team + records; then evict caches and write audit rows. Keys of users off the roster and the teams' + models stay. ``team_ids`` names only the teams this call deleted. ``before_commit`` runs on the + delete's transaction with the members of the rosters read under the lock, so its writes commit + or roll back with the delete. + """ + from litellm.proxy.proxy_server import ( + proxy_logging_obj, + user_api_key_cache, + ) + + team_ids: Final = [team.team_id for team in teams] + ## DELETE TEAMS # Both the delete and the reconcile sweep run under every team's advisory lock # (TEAM_ADVISORY_LOCK_SQL, the same one /team/member_add takes before its own writes), @@ -4674,14 +4713,28 @@ async def delete_team( # the lock before this transaction starts, in which case this sweep reaches what it wrote, # or is still waiting on the lock, in which case its own re-read happens after this commits # and sees the row gone before it writes anything. - delete_filter: Final[_TeamIdInFilter] = {"team_id": {"in": data.team_ids}} - async with prisma_client.tx(timeout=timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS)) as tx: - for team_id in sorted(data.team_ids): + tx_budget: Final = timedelta(seconds=TEAM_DELETE_TRANSACTION_TIMEOUT_SECONDS) + async with prisma_client.tx(max_wait=tx_budget, timeout=tx_budget) as tx: + for team_id in sorted(team_ids): await tx.query_raw(TEAM_ADVISORY_LOCK_SQL, team_id) - await tx.litellm_teamtable.delete_many(where=delete_filter) - await _sweep_deleted_team_references_tx(team_ids=data.team_ids, tx=tx) + team_tx: Final = _team_delete_tx(tx) + locked: Final = await _locked_teams_tx(team_ids=team_ids, tx=team_tx) + member_keys, jwt_mapping_cache_keys = await _delete_roster_member_keys_tx( + locked=locked, + tx=tx, + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + await _delete_locked_teams_tx( + locked=locked, tx=team_tx, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by + ) + detached_per_team: Final = await _sweep_deleted_team_references_tx(team_ids=team_ids, tx=team_tx) + if before_commit is not None: + await before_commit(tx, _locked_roster_user_ids(locked)) - deleted_teams: Final[_DeletedTeamsResult] = {"deleted_teams": data.team_ids} + rostered_per_team: Final = await _resolve_deleted_team_member_user_ids(teams=locked, prisma_client=prisma_client) + member_ids_per_team: Final = _members_per_deleted_team(rostered_per_team, detached_per_team) # Evict AFTER the rows are gone. Both writers of these keys (`_cache_team_object` and # `get_team_object_by_alias`) hydrate from the db, so evicting first leaves a window where a @@ -4689,7 +4742,7 @@ async def delete_team( # invalidated anything. Nothing fallible runs between the delete and this, or a failure there # would strand the deleted team in cache. await _invalidate_deleted_team_cache( - teams=team_rows, + teams=locked, user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) @@ -4697,44 +4750,167 @@ async def delete_team( member_ids_per_team=member_ids_per_team, user_api_key_cache=user_api_key_cache, ) + await _invalidate_deleted_key_cache( + keys=member_keys, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) - for deleted_team in team_rows: + _schedule_deleted_team_audit_rows( + teams=locked, user_api_key_dict=user_api_key_dict, litellm_changed_by=litellm_changed_by + ) + if member_keys: + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=member_keys, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + + for deleted_team in locked: _emit_team_members_metric(deleted_team.model_copy(update={"members_with_roles": ()})) await sync_team_access_group_membership(prisma_client=prisma_client, team_id=deleted_team.team_id) - return deleted_teams + return DeletedTeams(team_ids=tuple(team.team_id for team in locked)) -async def _sweep_deleted_team_references(team_ids: Sequence[str], prisma_client: PrismaClient) -> None: +def _locked_roster_user_ids(locked: Sequence[LiteLLM_TeamTable]) -> tuple[str, ...]: + return tuple(dict.fromkeys(chain.from_iterable(_roster_user_ids(team) for team in locked))) + + +def _members_per_deleted_team( + rostered: Sequence[tuple[str, Sequence[str]]], detached: Sequence[tuple[str, Sequence[str]]] +) -> tuple[tuple[str, tuple[str, ...]], ...]: + detached_by_team: Final = MappingProxyType({team_id: user_ids for team_id, user_ids in detached}) + return tuple( + (team_id, tuple(dict.fromkeys((*user_ids, *detached_by_team[team_id])))) for team_id, user_ids in rostered + ) + + +def _roster_user_ids(team: LiteLLM_TeamTable) -> tuple[str, ...]: + return tuple(member.user_id for member in team.members_with_roles if member.user_id is not None) + + +def _is_roster_member_key( + key: "prisma_models.LiteLLM_VerificationToken", roster_by_team: Mapping[str, frozenset[str]] +) -> bool: + return key.team_id is not None and key.user_id in roster_by_team.get(key.team_id, frozenset()) + + +async def _locked_teams_tx(team_ids: Sequence[str], tx: _TeamDeleteTx) -> tuple[LiteLLM_TeamTable, ...]: + rows: Final = await find_many_in(tx.litellm_teamtable, "team_id", team_ids) + return tuple(LiteLLM_TeamTable.model_validate(row.model_dump()) for row in rows) + + +async def _delete_roster_member_keys_tx( + locked: Sequence[LiteLLM_TeamTable], + tx: "Prisma", + prisma_client: PrismaClient, + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None, +) -> tuple[tuple["prisma_models.LiteLLM_VerificationToken", ...], tuple[str, ...]]: + from litellm.proxy.management_endpoints.key_management_endpoints import ( + persist_deleted_verification_tokens, + ) + + team_tx: Final = _team_delete_tx(tx) + roster_by_team: Final = MappingProxyType({team.team_id: frozenset(_roster_user_ids(team)) for team in locked}) + team_keys: Final = await find_many_in(team_tx.litellm_verificationtoken, "team_id", tuple(roster_by_team)) + member_keys: Final = tuple(key for key in team_keys if _is_roster_member_key(key, roster_by_team)) + if not member_keys: + return (), () + tokens: Final = tuple(key.token for key in member_keys) + jwt_mappings: Final = await find_many_in(team_tx.litellm_jwtkeymapping, "token", tokens) + await persist_deleted_verification_tokens( + keys=member_keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken + prisma_client=prisma_client, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + tx=tx, + ) + await delete_many_in(team_tx.litellm_verificationtoken, "token", tokens, atomicity="caller_transaction") + return member_keys, tuple( + jwt_key_mapping_cache_key(mapping.jwt_claim_name, mapping.jwt_claim_value, mapping.jwt_issuer) + for mapping in jwt_mappings + ) + + +async def _delete_locked_teams_tx( + locked: Sequence[LiteLLM_TeamTable], + tx: _TeamDeleteTx, + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None, +) -> None: + await _persist_deleted_team_records( + teams=locked, + deleted_teams=tx.litellm_deletedteamtable, + user_api_key_dict=user_api_key_dict, + litellm_changed_by=litellm_changed_by, + ) + await delete_many_in( + tx.litellm_teamtable, + "team_id", + tuple(team.team_id for team in locked), + atomicity="caller_transaction", + ) + + +def _schedule_deleted_team_audit_rows( + teams: Sequence[LiteLLM_TeamTable], + user_api_key_dict: UserAPIKeyAuth, + litellm_changed_by: str | None, +) -> None: + from litellm.proxy.management_helpers.audit_logs import ( + get_audit_log_changed_by, + is_audit_logging_enabled, + ) + from litellm.proxy.proxy_server import ( + create_audit_log_for_update, + litellm_proxy_admin_name, + ) + + if not is_audit_logging_enabled(): + return + for team in teams: + asyncio.create_task( + create_audit_log_for_update( + request_data=LiteLLM_AuditLogs( + id=str(uuid.uuid4()), + updated_at=datetime.now(timezone.utc), + changed_by=get_audit_log_changed_by( + litellm_changed_by=litellm_changed_by, + user_api_key_dict=user_api_key_dict, + litellm_proxy_admin_name=litellm_proxy_admin_name, + ), + changed_by_api_key=user_api_key_dict.api_key, + table_name=LitellmTableNames.TEAM_TABLE_NAME, + object_id=team.team_id, + action="deleted", + updated_values="{}", + before_value=team.model_dump_json(exclude_none=True), + ) + ) + ) + + +async def _sweep_deleted_team_references_tx( + team_ids: Sequence[str], tx: _TeamDeleteTx +) -> tuple[tuple[str, tuple[str, ...]], ...]: """ - Strip the deleted team ids from every user row and team-membership row that still references them. - - The per-member `team_member_delete` pass above only reaches users listed in the team's - `members_with_roles`, so a user row that outlived its roster entry is invisible to it and keeps - surfacing the team on `/user/info` after the team is gone. - - #36839 closed the route that created that drift, by resolving member removal off the roster - entry's `user_id` rather than the identifier the caller happened to pass. It does not backfill - rows that already drifted, which is the state this was reported against, so the sweep still has - to run on delete. - - One statement filters the array in place rather than read-filter-write: rewriting the whole - array from a snapshot read outside a transaction drops any team a concurrent `/team/member_add` - appended in between. It locks the user rows in the order `USER_ROWS_LOCK_SQL` takes them, so it - never waits on a SCIM group push in a cycle. + Strip the deleted team ids from every user and team-membership row still referencing them, on + the transaction holding the teams' advisory locks. User rows are locked first in + ``USER_ROWS_LOCK_SQL`` order so the sweep never deadlocks with a SCIM group push, and one + statement filters each array in place so a team a concurrent ``/team/member_add`` appended is + kept. Returns, per team, the users read under the lock as referencing it. """ - _ = await prisma_client.db.execute_raw(_STRIP_DELETED_TEAMS_FROM_USERS_SQL, list(team_ids)) - - _ = await _team_membership_db(prisma_client).delete_many(where=_TeamIdInFilter(team_id={"in": tuple(team_ids)})) - - -async def _sweep_deleted_team_references_tx(team_ids: Sequence[str], tx: _TeamDeleteTx) -> None: - """Same sweep as `_sweep_deleted_team_references`, run on the transaction that holds - every id's advisory lock and deletes the team rows, so it commits or rolls back with them.""" + locked: Final = _DetachedMembers.validate_python(await tx.query_raw(_LOCK_DELETED_TEAM_MEMBERS_SQL, list(team_ids))) _ = await tx.execute_raw(_STRIP_DELETED_TEAMS_FROM_USERS_SQL, list(team_ids)) membership_filter: Final[_TeamIdInFilter] = {"team_id": {"in": tuple(team_ids)}} _ = await tx.litellm_teammembership.delete_many(where=membership_filter) + return tuple( + (team_id, tuple(member.user_id for member in locked if team_id in member.teams)) for team_id in team_ids + ) async def _invalidate_deleted_key_cache( @@ -4791,17 +4967,20 @@ async def _evict_deleted_team_member_cache( member_user_ids: Sequence[str], user_api_key_cache: UserApiKeyCache, ) -> None: - await evict_and_broadcast(cache_keys=tuple(member_user_ids), user_api_key_cache=user_api_key_cache) - await asyncio.gather( - *( - invalidate_team_member_spend_state( - user_id=user_id, - team_id=team_id, - user_api_key_cache=user_api_key_cache, - ) - for user_id in member_user_ids - ) + _ = await asyncio.gather( + *(await_inflight_team_membership_load(user_id=user_id, team_id=team_id) for user_id in member_user_ids) ) + cache_keys: Final = tuple(member_cache_keys(team_id, member_user_ids)) + evict_local(cache_keys=cache_keys, user_api_key_cache=user_api_key_cache) + await evict_shared(cache_keys=cache_keys, cache=user_api_key_cache) + await broadcast_evictions(cache_keys=cache_keys) + + +def member_cache_keys(team_id: str, user_ids: Sequence[str]) -> Iterator[str]: + for user_id in user_ids: + yield user_id + yield team_membership_auth_cache_key(team_id=team_id, user_id=user_id) + yield team_membership_reservation_cache_key(user_id=user_id, team_id=team_id) async def _resolve_deleted_team_member_user_ids( @@ -4832,7 +5011,7 @@ async def _deleted_team_member_user_ids(team: LiteLLM_TeamTable, prisma_client: def _transform_teams_to_deleted_records( - teams: list[LiteLLM_TeamTable], + teams: Sequence[LiteLLM_TeamTable], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None = None, ) -> list[dict[str, object]]: @@ -4881,17 +5060,17 @@ def _transform_teams_to_deleted_records( async def _save_deleted_team_records( records: list[dict[str, object]], - prisma_client: PrismaClient, + deleted_teams: "TableActions[prisma_models.LiteLLM_DeletedTeamTable]", ) -> None: """Save deleted team records to the database.""" if not records: return - await _deleted_team_db(prisma_client).create_many(data=records) + await deleted_teams.create_many(data=records) async def _persist_deleted_team_records( - teams: list[LiteLLM_TeamTable], - prisma_client: PrismaClient, + teams: Sequence[LiteLLM_TeamTable], + deleted_teams: "TableActions[prisma_models.LiteLLM_DeletedTeamTable]", user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None = None, ) -> None: @@ -4903,7 +5082,7 @@ async def _persist_deleted_team_records( ) await _save_deleted_team_records( records=records, - prisma_client=prisma_client, + deleted_teams=deleted_teams, ) diff --git a/litellm/proxy/management_helpers/audit_logs.py b/litellm/proxy/management_helpers/audit_logs.py index ef95e8e2c0f..aa31f44dd13 100644 --- a/litellm/proxy/management_helpers/audit_logs.py +++ b/litellm/proxy/management_helpers/audit_logs.py @@ -4,6 +4,7 @@ Functions to create audit logs for LiteLLM Proxy import asyncio import json +from collections.abc import Awaitable, Sequence from datetime import datetime, timezone from typing import Final, TypeVar @@ -258,12 +259,19 @@ async def create_object_audit_log( ) -async def _insert_audit_log_row(repository: AuditLogRepository, request_data: LiteLLM_AuditLogs) -> None: +async def _insert_audit_logs(insert: Awaitable[object], description: str) -> None: try: - await repository.table.create(data={**request_data.model_dump(exclude_none=True)}) + await insert except Exception as e: # [Non-Blocking Exception. Do not allow blocking LLM API call] - verbose_proxy_logger.error("Failed Creating audit log %s", e) + verbose_proxy_logger.error("Failed Creating %s %s", description, e) + + +def _json_encoded(request_data: LiteLLM_AuditLogs) -> LiteLLM_AuditLogs: + values: Final = (("updated_values", request_data.updated_values), ("before_value", request_data.before_value)) + return request_data.model_copy( + update={field: json.dumps(value) for field, value in values if isinstance(value, dict)} + ) async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): @@ -279,15 +287,10 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): return verbose_proxy_logger.debug("creating audit log for %s", request_data) - - if isinstance(request_data.updated_values, dict): - request_data.updated_values = json.dumps(request_data.updated_values) - - if isinstance(request_data.before_value, dict): - request_data.before_value = json.dumps(request_data.before_value) + encoded: Final = _json_encoded(request_data) # Dispatch to external audit log callbacks regardless of DB availability - await _dispatch_audit_log_to_callbacks(request_data) + await _dispatch_audit_log_to_callbacks(encoded) if prisma_client is None: verbose_proxy_logger.error("prisma_client is None, cannot write audit log to DB") @@ -295,7 +298,41 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs): await track_audit_task( asyncio.create_task( - _insert_audit_log_row(AuditLogRepository(prisma_client), request_data), - name=_audit_task_name(request_data, "the database"), + _insert_audit_logs( + AuditLogRepository(prisma_client).table.create(data={**encoded.model_dump(exclude_none=True)}), + "audit log", + ), + name=_audit_task_name(encoded, "the database"), + ) + ) + + +async def create_audit_logs_for_updates(rows: Sequence[LiteLLM_AuditLogs]) -> None: + """Create audit logs for many objects with one insert.""" + if not rows or not is_audit_logging_enabled(): + return + + from litellm.proxy.proxy_server import premium_user, prisma_client + + if premium_user is not True: + return + + encoded: Final = tuple(_json_encoded(row) for row in rows) + for row in encoded: + await _dispatch_audit_log_to_callbacks(row) + + if prisma_client is None: + verbose_proxy_logger.error("prisma_client is None, cannot write audit log to DB") + return + + await track_audit_task( + asyncio.create_task( + _insert_audit_logs( + AuditLogRepository(prisma_client).table.create_many( + data=tuple(row.model_dump(exclude_none=True) for row in encoded) + ), + f"{len(encoded)} audit logs", + ), + name=f"{len(encoded)} audit logs to the database", ) ) diff --git a/litellm/proxy/management_helpers/team_roster_sync.py b/litellm/proxy/management_helpers/team_roster_sync.py index 4b29d4676f8..dc0ba2e47d4 100644 --- a/litellm/proxy/management_helpers/team_roster_sync.py +++ b/litellm/proxy/management_helpers/team_roster_sync.py @@ -1,6 +1,6 @@ import asyncio import json -from collections.abc import Iterator, Mapping, Sequence +from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import timedelta from types import MappingProxyType @@ -23,8 +23,6 @@ from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import ( ) from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, - team_membership_auth_cache_key, - team_membership_reservation_cache_key, ) from litellm.proxy.hooks.key_management_event_hooks import KeyManagementEventHooks from litellm.proxy.management_endpoints.key_management_endpoints import ( @@ -33,6 +31,7 @@ from litellm.proxy.management_endpoints.key_management_endpoints import ( from litellm.proxy.management_endpoints.team_endpoints import ( _emit_team_members_metric, # pyright: ignore[reportPrivateUsage] # same gauge /team/member_add moves _schedule_team_membership_audit_log, # pyright: ignore[reportPrivateUsage] # same audit row /team/member_add writes + member_cache_keys, ) from litellm.proxy.management_helpers.access_group_team_sync import ( DETACH_TEAM_FROM_USERS_SQL, @@ -277,7 +276,7 @@ async def _settle_caches( evicted: Final = ( *deleted_tokens, *removal.jwt_mapping_cache_keys, - *_member_cache_keys(team.team_id, changed_user_ids), + *member_cache_keys(team.team_id, changed_user_ids), ) evict_local(cache_keys=evicted, user_api_key_cache=user_api_key_cache) await evict_shared(cache_keys=evicted, cache=user_api_key_cache) @@ -292,13 +291,6 @@ async def _settle_caches( await broadcast_evictions(cache_keys=evicted) -def _member_cache_keys(team_id: str, user_ids: Sequence[str]) -> Iterator[str]: - for user_id in user_ids: - yield user_id - yield team_membership_auth_cache_key(team_id=team_id, user_id=user_id) - yield team_membership_reservation_cache_key(user_id=user_id, team_id=team_id) - - async def sync_team_roster( prisma_client: PrismaClient, team_id: str, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index b5101898bb1..ea1044d4a55 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -4832,6 +4832,7 @@ def jsonify_object(data: dict) -> dict: _deprecated_key_cache: Final[LimitedSizeOrderedDict] = LimitedSizeOrderedDict(max_size=1000) _DEPRECATED_KEY_CACHE_TTL_SECONDS: Final = 60 _PRISMA_DEFAULT_TX_TIMEOUT: Final = timedelta(seconds=5) +_PRISMA_DEFAULT_TX_MAX_WAIT: Final = timedelta(seconds=2) async def _lookup_deprecated_key( @@ -5238,13 +5239,17 @@ class PrismaClient: return self.db.read_target return self.db - def tx(self, *, timeout: timedelta = _PRISMA_DEFAULT_TX_TIMEOUT) -> "TransactionManager": + def tx( + self, *, timeout: timedelta = _PRISMA_DEFAULT_TX_TIMEOUT, max_wait: timedelta = _PRISMA_DEFAULT_TX_MAX_WAIT + ) -> "TransactionManager": """Open an interactive transaction on the writer. Callers go through this instead of reaching into ``self.db`` so writer selection and read-replica routing stay encapsulated in the wrapper. """ - return cast("TransactionManager", self.db.tx(timeout=timeout)) # cast-ok: untyped __getattr__ delegate + return cast( # cast-ok: untyped __getattr__ delegate + "TransactionManager", self.db.tx(max_wait=max_wait, timeout=timeout) + ) def get_request_status(self, payload: dict | SpendLogsPayload) -> Literal["success", "failure"]: """ diff --git a/tests/code_coverage_tests/unbounded_in_baseline.txt b/tests/code_coverage_tests/unbounded_in_baseline.txt index c7e73bf921c..1286ea25c4e 100644 --- a/tests/code_coverage_tests/unbounded_in_baseline.txt +++ b/tests/code_coverage_tests/unbounded_in_baseline.txt @@ -80,14 +80,12 @@ litellm/proxy/management_endpoints/team_endpoints.py _get_keys_count_by_team pri litellm/proxy/management_endpoints/team_endpoints.py _hydrate_member_user_details prisma user_id.in `sorted(user_ids)` 0 litellm/proxy/management_endpoints/team_endpoints.py _resolve_existing_member_user_ids prisma user_id.in `sorted(requested_user_ids)` 0 litellm/proxy/management_endpoints/team_endpoints.py resolve_team_daily_activity_scope prisma team_id.in `list(team_ids_list)` 0 -litellm/proxy/management_endpoints/team_endpoints.py _sweep_deleted_team_references prisma team_id.in `tuple(team_ids)` 0 litellm/proxy/management_endpoints/team_endpoints.py _sweep_deleted_team_references_tx prisma team_id.in `tuple(team_ids)` 0 litellm/proxy/management_endpoints/team_endpoints.py _team_member_delete prisma user_id.in `sorted(addressed_user_ids)` 0 litellm/proxy/management_endpoints/team_endpoints.py _team_member_delete prisma user_id.in `sorted(user_ids_to_delete)` 0 litellm/proxy/management_endpoints/team_endpoints.py _team_member_delete prisma user_id.in `sorted(user_ids_to_delete)` 1 litellm/proxy/management_endpoints/team_endpoints.py _team_user_spend_sql raw-sql sl.team_id.IN `IN ({team_placeholders})` 0 -litellm/proxy/management_endpoints/team_endpoints.py delete_team prisma team_id.in `data.team_ids` 0 -litellm/proxy/management_endpoints/team_endpoints.py delete_team prisma team_id.in `data.team_ids` 1 +litellm/proxy/management_endpoints/team_endpoints.py delete_validated_teams prisma team_id.in `team_ids` 0 litellm/proxy/management_endpoints/team_endpoints.py get_all_team_memberships prisma team_id.in `team_ids` 0 litellm/proxy/management_endpoints/team_endpoints.py list_available_teams prisma team_id.in `available_teams` 0 litellm/proxy/management_endpoints/tool_management_endpoints.py get_tool_spend prisma tool_name.in `[row.tool_name for row in top_tools]` 0 diff --git a/tests/integration/_support/scim.py b/tests/integration/_support/scim.py new file mode 100644 index 00000000000..11eea030ba3 --- /dev/null +++ b/tests/integration/_support/scim.py @@ -0,0 +1,207 @@ +"""SCIM v2 group helpers for the management cells: seeded users, group pushes, the team lock, and what a group +leaves behind in Postgres.""" + +import os +import uuid +from collections.abc import Generator, Mapping, Sequence +from contextlib import contextmanager +from typing import Final + +import httpx +import psycopg +from integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + delete_key_if_present, + object_value, + string_value, +) +from integration._support.database import read_rows, write_rows +from pydantic import JsonValue + +GROUP_SCHEMA: Final = "urn:ietf:params:scim:schemas:core:2.0:Group" +PATCH_OP_SCHEMA: Final = "urn:ietf:params:scim:api:messages:2.0:PatchOp" +SEED_USERS_SQL: Final = """ +INSERT INTO "LiteLLM_UserTable" (user_id, user_role, teams, models) +SELECT %s || '-' || lpad(n::text, 3, '0'), 'internal_user', '{}'::text[], '{}'::text[] +FROM generate_series(1, %s::int) AS n +""" +TAKE_TEAM_LOCK_SQL: Final = "SELECT pg_advisory_xact_lock(hashtext(%s))" +WAITERS_ON_HELD_LOCK_SQL: Final = """ +SELECT count(*)::text AS waiting +FROM pg_locks waiter +JOIN pg_stat_activity session ON session.pid = waiter.pid +WHERE waiter.locktype = 'advisory' + AND NOT waiter.granted + AND session.wait_event_type = 'Lock' + AND (waiter.classid, waiter.objid, waiter.objsubid) IN ( + SELECT held.classid, held.objid, held.objsubid + FROM pg_locks held + WHERE held.locktype = 'advisory' AND held.granted AND held.pid = %s::int + ) +""" +TAKE_USER_ROW_LOCK_SQL: Final = 'SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s FOR UPDATE' +WAITERS_ON_ROWS_LOCKED_BY_SQL: Final = """ +SELECT count(*)::text AS waiting +FROM pg_locks waiter +WHERE waiter.locktype = 'transactionid' + AND NOT waiter.granted + AND waiter.transactionid IN ( + SELECT held.transactionid + FROM pg_locks held + WHERE held.locktype = 'transactionid' AND held.granted AND held.pid = %s::int + ) +""" + + +def seed_users(scenario: Scenario, count: int) -> tuple[str, ...]: + prefix: Final = f"integration-scim-{uuid.uuid4().hex}" + write_rows(SEED_USERS_SQL, (prefix, str(count))) + scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_UserTable" WHERE user_id LIKE %s', (f"{prefix}-%",)) + return tuple(f"{prefix}-{index:03d}" for index in range(1, count + 1)) + + +def members(users: Sequence[str]) -> list[JsonValue]: + return [{"value": user} for user in users] + + +def group_name() -> str: + return f"integration-{uuid.uuid4().hex}" + + +def create_group(candidate: Gateway, users: Sequence[str], *, display_name: str | None = None) -> httpx.Response: + return candidate.request( + "POST", + "/scim/v2/Groups", + {"schemas": [GROUP_SCHEMA], "displayName": display_name or group_name(), "members": members(users)}, + ) + + +def replace_group( + candidate: Gateway, team: str, users: Sequence[str], *, display_name: str | None = None +) -> httpx.Response: + return candidate.request( + "PUT", + f"/scim/v2/Groups/{team}", + {"schemas": [GROUP_SCHEMA], "id": team, "displayName": display_name or group_name(), "members": members(users)}, + ) + + +def add_to_group(candidate: Gateway, team: str, users: Sequence[str]) -> httpx.Response: + return candidate.request( + "PATCH", + f"/scim/v2/Groups/{team}", + {"schemas": [PATCH_OP_SCHEMA], "Operations": [{"op": "add", "path": "members", "value": members(users)}]}, + ) + + +def body(response: httpx.Response) -> dict[str, JsonValue]: + return JSON_OBJECT.validate_json(response.content) + + +def member_ids(group: Mapping[str, JsonValue]) -> frozenset[str]: + listed: Final = group.get("members") or [] + assert isinstance(listed, list), listed + return frozenset(string_value(object_value(member)["value"]) for member in listed) + + +def delete_group(candidate: Gateway, team: str, *, key: str | None = None) -> httpx.Response: + return candidate.request("DELETE", f"/scim/v2/Groups/{team}", key=key) + + +def delete_team_if_present(scenario: Scenario, team: str) -> None: + if read_rows('SELECT team_id FROM "LiteLLM_TeamTable" WHERE team_id = %s', (team,)): + scenario.delete_team(team) + + +def created_team(scenario: Scenario, created: httpx.Response) -> str: + assert created.status_code == 201, created.text + team: Final = string_value(body(created)["id"]) + scenario.cleanups.callback(delete_team_if_present, scenario, team) + return team + + +def team_key(candidate: Gateway, team: str, user: str) -> str: + created: Final = candidate.post( + "/key/generate", {"team_id": team, "user_id": user, "key_alias": f"integration-{uuid.uuid4().hex}"} + ) + return string_value(created["key"]) + + +def service_account_key(candidate: Gateway, scenario: Scenario, team: str) -> str: + created: Final = candidate.post("/key/generate", {"team_id": team, "key_alias": f"integration-{uuid.uuid4().hex}"}) + key: Final = string_value(created["key"]) + scenario.cleanups.callback(delete_key_if_present, candidate, key) + return key + + +def membership_user_ids(team: str) -> frozenset[str]: + rows: Final = read_rows('SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s', (team,)) + return frozenset(string_value(row["user_id"]) for row in rows) + + +def keys_of(team: str) -> frozenset[str]: + rows: Final = read_rows('SELECT token FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)) + return frozenset(string_value(row["token"]) for row in rows) + + +def users_referencing(team: str) -> frozenset[str]: + rows: Final = read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE %s = ANY(teams)', (team,)) + return frozenset(string_value(row["user_id"]) for row in rows) + + +def user_role(candidate: Gateway, user: str) -> str: + return string_value(object_value(candidate.get("/user/info", {"user_id": user})["user_info"])["user_role"]) + + +def groups_of(candidate: Gateway, user: str) -> frozenset[str]: + groups: Final = candidate.get(f"/scim/v2/Users/{user}").get("groups") or [] + assert isinstance(groups, list), groups + return frozenset(string_value(object_value(group)["value"]) for group in groups) + + +def assert_landed(candidate: Gateway, team: str, users: Sequence[str]) -> None: + assert member_ids(candidate.get(f"/scim/v2/Groups/{team}")) == frozenset(users) + assert membership_user_ids(team) == frozenset(users) + assert users_referencing(team) == frozenset(users) + + +def assert_gone(candidate: Gateway, team: str, users: Sequence[str], key: str) -> None: + assert candidate.request("GET", f"/scim/v2/Groups/{team}").status_code == 404 + assert team not in groups_of(candidate, users[0]) + assert membership_user_ids(team) == frozenset() + assert users_referencing(team) == frozenset() + assert keys_of(team) == frozenset() + assert candidate.request("GET", "/v1/models", key=key).status_code == 401 + + +@contextmanager +def held_team_lock(team: str) -> Generator[int]: + """Hold the team's advisory lock from a session of our own and yield its backend pid; leaving the block + commits, which releases the lock to whoever is queued on it.""" + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(TAKE_TEAM_LOCK_SQL, (team,)) + yield holder.info.backend_pid + + +@contextmanager +def held_user_row(user: str) -> Generator[int]: + """Hold a row lock on the user's row from a session of our own and yield its backend pid, so a role write + that reaches the row waits until leaving the block commits.""" + with psycopg.connect(os.environ["DATABASE_URL"]) as holder: + holder.execute(TAKE_USER_ROW_LOCK_SQL, (user,)) + yield holder.info.backend_pid + + +def waiters_on_lock_held_by(backend_pid: int) -> int: + return _waiting(WAITERS_ON_HELD_LOCK_SQL, backend_pid) + + +def waiters_on_rows_locked_by(backend_pid: int) -> int: + return _waiting(WAITERS_ON_ROWS_LOCKED_BY_SQL, backend_pid) + + +def _waiting(query: str, backend_pid: int) -> int: + rows: Final = read_rows(query, (str(backend_pid),)) + return int(string_value(rows[0]["waiting"])) diff --git a/tests/integration/management/test_scim_group_bulk_roster.py b/tests/integration/management/test_scim_group_bulk_roster.py index 6d8e657c5ce..c61aa62f52e 100644 --- a/tests/integration/management/test_scim_group_bulk_roster.py +++ b/tests/integration/management/test_scim_group_bulk_roster.py @@ -3,7 +3,10 @@ POST /scim/v2/Groups, PUT and PATCH /scim/v2/Groups/{id} used to run /team/member_add once per member, so a 500-member push was thousands of statements and outlasted the edge in front of the proxy. One locked transaction now resolves every member, writes the roster and the membership rows and tags the users, so a -push of 500 members runs as many statements as a push of 5. +push of 500 members runs as many statements as a push of 5. DELETE /scim/v2/Groups/{id} used to detach the +members one at a time and leave their membership rows and keys behind; it now detaches them in one locked pass, +so deleting 500 members who each hold a key costs as many statements as deleting 5 and leaves no membership rows +or member keys. """ import os @@ -13,6 +16,7 @@ from collections.abc import Callable, Iterator, Mapping, Sequence from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager from dataclasses import dataclass +from functools import partial from hashlib import sha256 from pathlib import Path from typing import Final, LiteralString @@ -22,7 +26,6 @@ import psycopg import pytest from integration._support.budget import case_timeout_seconds from integration._support.client import ( - JSON_OBJECT, Gateway, Scenario, delete_key_if_present, @@ -32,14 +35,30 @@ from integration._support.client import ( ) from integration._support.database import read_rows, write_rows from integration._support.database_relay import StatementCountingRelay, statement_counting_relay -from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.scim import ( + GROUP_SCHEMA, + PATCH_OP_SCHEMA, + add_to_group, + assert_gone, + assert_landed, + body, + create_group, + created_team, + delete_group, + member_ids, + members, + membership_user_ids, + replace_group, + seed_users, + team_key, + users_referencing, +) from psycopg import sql from pydantic import JsonValue pytestmark: Final = pytest.mark.shares_worker_state -GROUP_SCHEMA: Final = "urn:ietf:params:scim:schemas:core:2.0:Group" -PATCH_OP_SCHEMA: Final = "urn:ietf:params:scim:api:messages:2.0:PatchOp" SMALL_PUSH: Final = 5 LARGE_PUSH: Final = 500 # Background work on an idle proxy (the cron-job leader poll, spend flushes) lands a few statements inside a @@ -48,7 +67,9 @@ NOISE_ALLOWANCE: Final = 100 UNFIXED_PUSH_SECONDS: Final = 120 SWAP_GROUP: Final = 200 SWAP_ROUNDS: Final = 10 +OWNED_PROXY_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 2 * UNFIXED_PUSH_SECONDS) CHAOS_GROUP: Final = 100 +KEY_WRITERS: Final = 16 ROLLBACK_SECONDS: Final = 90 TEAM_ROW_SQL: Final = 'SELECT members_with_roles FROM "LiteLLM_TeamTable" WHERE team_id = %s' LOCK_USER_ROW_SQL: Final = 'SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s FOR UPDATE' @@ -56,16 +77,12 @@ LOCK_KEY_ROW_SQL: Final = 'SELECT token FROM "LiteLLM_VerificationToken" WHERE t SESSIONS_BLOCKED_BY_SQL: Final = ( "SELECT count(*)::int AS waiting FROM pg_stat_activity WHERE %s::int = ANY(pg_blocking_pids(pid))" ) +BACKENDS_BLOCKED_BY_SQL: Final = "SELECT pid FROM pg_stat_activity WHERE %s::int = ANY(pg_blocking_pids(pid))" TRANSACTIONS_OPEN_SINCE_SQL: Final = """ SELECT count(*)::int AS open FROM pg_stat_activity WHERE datname = current_database() AND backend_type = 'client backend' AND pid <> pg_backend_pid() AND xact_start < %s::timestamptz """ -SEED_USERS_SQL: Final = """ -INSERT INTO "LiteLLM_UserTable" (user_id, user_role, teams, models) -SELECT %s || '-' || lpad(n::text, 3, '0'), 'internal_user', '{}'::text[], '{}'::text[] -FROM generate_series(1, %s::int) AS n -""" @dataclass(frozen=True, slots=True) @@ -82,71 +99,6 @@ def _measured(relay: StatementCountingRelay, send: Callable[[], httpx.Response]) return Measured(response=response, statements=relay.statements - before, seconds=time.perf_counter() - started) -def _seed_users(scenario: Scenario, count: int) -> tuple[str, ...]: - prefix: Final = f"integration-scim-{uuid.uuid4().hex}" - write_rows(SEED_USERS_SQL, (prefix, str(count))) - scenario.cleanups.callback(write_rows, 'DELETE FROM "LiteLLM_UserTable" WHERE user_id LIKE %s', (f"{prefix}-%",)) - return tuple(f"{prefix}-{index:03d}" for index in range(1, count + 1)) - - -def _members(users: Sequence[str]) -> list[JsonValue]: - return [{"value": user} for user in users] - - -def _create_group(candidate: Gateway, users: Sequence[str]) -> httpx.Response: - return candidate.request( - "POST", - "/scim/v2/Groups", - {"schemas": [GROUP_SCHEMA], "displayName": f"integration-{uuid.uuid4().hex}", "members": _members(users)}, - ) - - -def _replace_group(candidate: Gateway, team: str, users: Sequence[str]) -> httpx.Response: - return candidate.request( - "PUT", - f"/scim/v2/Groups/{team}", - { - "schemas": [GROUP_SCHEMA], - "id": team, - "displayName": f"integration-{uuid.uuid4().hex}", - "members": _members(users), - }, - ) - - -def _add_to_group(candidate: Gateway, team: str, users: Sequence[str]) -> httpx.Response: - return candidate.request( - "PATCH", - f"/scim/v2/Groups/{team}", - { - "schemas": [PATCH_OP_SCHEMA], - "Operations": [{"op": "add", "path": "members", "value": _members(users)}], - }, - ) - - -def _body(response: httpx.Response) -> dict[str, JsonValue]: - return JSON_OBJECT.validate_json(response.content) - - -def _member_ids(body: Mapping[str, JsonValue]) -> frozenset[str]: - members: Final = body.get("members") or [] - assert isinstance(members, list), members - return frozenset(string_value(object_value(member)["value"]) for member in members) - - -def _created_team(scenario: Scenario, created: httpx.Response) -> str: - assert created.status_code == 201, created.text - team: Final = string_value(_body(created)["id"]) - scenario.cleanups.callback(scenario.delete_team, team) - return team - - -def _membership_user_ids(team: str) -> frozenset[str]: - rows: Final = read_rows('SELECT user_id FROM "LiteLLM_TeamMembership" WHERE team_id = %s', (team,)) - return frozenset(string_value(row["user_id"]) for row in rows) - - _UTC_MICROSECONDS: Final = "YYYY-MM-DD HH24:MI:SS.US" @@ -163,18 +115,7 @@ def _database_utc_now() -> str: return string_value(rows[0]["utc_now"]) -def _users_referencing(team: str) -> frozenset[str]: - rows: Final = read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE %s = ANY(teams)', (team,)) - return frozenset(string_value(row["user_id"]) for row in rows) - - -def _assert_landed(candidate: Gateway, team: str, users: Sequence[str]) -> None: - assert _member_ids(candidate.get(f"/scim/v2/Groups/{team}")) == frozenset(users) - assert _membership_user_ids(team) == frozenset(users) - assert _users_referencing(team) == frozenset(users) - - -@pytest.mark.timeout(300) # owned proxy boot plus three 500-member pushes +@pytest.mark.timeout(OWNED_PROXY_TIMEOUT) def test_group_pushes_of_500_members_run_as_many_statements_as_pushes_of_5( gateway: Gateway, tmp_path: Path, record_property: Callable[[str, object], None] ) -> None: @@ -191,39 +132,50 @@ def test_group_pushes_of_500_members_run_as_many_statements_as_pushes_of_5( ) as owned, ): candidate: Final = owned.gateway - small_a: Final = _seed_users(scenario, SMALL_PUSH) - small_b: Final = _seed_users(scenario, SMALL_PUSH) - large: Final = _seed_users(scenario, LARGE_PUSH) + small_a: Final = seed_users(scenario, SMALL_PUSH) + small_b: Final = seed_users(scenario, SMALL_PUSH) + large: Final = seed_users(scenario, LARGE_PUSH) - created_small: Final = _measured(relay, lambda: _create_group(candidate, small_a)) - small_team: Final = _created_team(scenario, created_small.response) - created_large: Final = _measured(relay, lambda: _create_group(candidate, large)) - large_team: Final = _created_team(scenario, created_large.response) - assert _member_ids(_body(created_large.response)) == frozenset(large) - _assert_landed(candidate, small_team, small_a) - _assert_landed(candidate, large_team, large) + created_small: Final = _measured(relay, lambda: create_group(candidate, small_a)) + small_team: Final = created_team(scenario, created_small.response) + created_large: Final = _measured(relay, lambda: create_group(candidate, large)) + large_team: Final = created_team(scenario, created_large.response) + assert member_ids(body(created_large.response)) == frozenset(large) + assert_landed(candidate, small_team, small_a) + assert_landed(candidate, large_team, large) detach_window_open: Final = _database_utc_now() - replaced_small: Final = _measured(relay, lambda: _replace_group(candidate, small_team, small_b)) - replaced_large: Final = _measured(relay, lambda: _replace_group(candidate, large_team, small_a)) + replaced_small: Final = _measured(relay, lambda: replace_group(candidate, small_team, small_b)) + replaced_large: Final = _measured(relay, lambda: replace_group(candidate, large_team, small_a)) assert replaced_small.response.status_code == 200, replaced_small.response.text assert replaced_large.response.status_code == 200, replaced_large.response.text - assert _member_ids(_body(replaced_small.response)) == frozenset(small_b) - _assert_landed(candidate, small_team, small_b) - _assert_landed(candidate, large_team, small_a) + assert member_ids(body(replaced_small.response)) == frozenset(small_b) + assert_landed(candidate, small_team, small_b) + assert_landed(candidate, large_team, small_a) assert detach_window_open <= _updated_at(large[0]) <= _database_utc_now() - patched_small: Final = _measured(relay, lambda: _add_to_group(candidate, small_team, small_a)) - patched_large: Final = _measured(relay, lambda: _add_to_group(candidate, large_team, large)) + patched_small: Final = _measured(relay, lambda: add_to_group(candidate, small_team, small_a)) + patched_large: Final = _measured(relay, lambda: add_to_group(candidate, large_team, large)) assert patched_small.response.status_code == 200, patched_small.response.text assert patched_large.response.status_code == 200, patched_large.response.text - _assert_landed(candidate, small_team, [*small_a, *small_b]) - _assert_landed(candidate, large_team, [*small_a, *large]) + assert_landed(candidate, small_team, [*small_a, *small_b]) + assert_landed(candidate, large_team, [*small_a, *large]) + + with ThreadPoolExecutor(max_workers=KEY_WRITERS) as pool: + small_keys: Final = tuple(pool.map(partial(team_key, candidate, small_team), (*small_a, *small_b))) + large_keys: Final = tuple(pool.map(partial(team_key, candidate, large_team), (*small_a, *large))) + deleted_small: Final = _measured(relay, lambda: delete_group(candidate, small_team)) + deleted_large: Final = _measured(relay, lambda: delete_group(candidate, large_team)) + assert deleted_small.response.status_code == 204, deleted_small.response.text + assert deleted_large.response.status_code == 204, deleted_large.response.text + assert_gone(candidate, small_team, [*small_a, *small_b], small_keys[0]) + assert_gone(candidate, large_team, [*small_a, *large], large_keys[-1]) pushes: Final = ( ("POST", created_small, created_large), ("PUT", replaced_small, replaced_large), ("PATCH", patched_small, patched_large), + ("DELETE", deleted_small, deleted_large), ) for verb, small, big in pushes: record_property( @@ -238,13 +190,13 @@ def test_group_pushes_of_500_members_run_as_many_statements_as_pushes_of_5( def _trade_rosters(candidate: Gateway, teams: tuple[str, str], rosters: tuple[Sequence[str], Sequence[str]]) -> None: with ThreadPoolExecutor(max_workers=2) as pool: - left: Final = pool.submit(_replace_group, candidate, teams[0], rosters[1]) - right: Final = pool.submit(_replace_group, candidate, teams[1], rosters[0]) + left: Final = pool.submit(replace_group, candidate, teams[0], rosters[1]) + right: Final = pool.submit(replace_group, candidate, teams[1], rosters[0]) for response in (left.result(), right.result()): assert response.status_code == 200, response.text -@pytest.mark.timeout(300) +@pytest.mark.timeout(OWNED_PROXY_TIMEOUT) def test_two_groups_trading_rosters_in_parallel_both_land(gateway: Gateway, tmp_path: Path) -> None: with ( gateway.scenario() as scenario, @@ -258,15 +210,18 @@ def test_two_groups_trading_rosters_in_parallel_both_land(gateway: Gateway, tmp_ ) as owned, ): candidate: Final = owned.gateway - left_users: Final = _seed_users(scenario, SWAP_GROUP) - right_users: Final = _seed_users(scenario, SWAP_GROUP) - left: Final = _created_team(scenario, _create_group(candidate, left_users)) - right: Final = _created_team(scenario, _create_group(candidate, right_users)) - for round_index in range(SWAP_ROUNDS): - held: Final = (left_users, right_users) if round_index % 2 == 0 else (right_users, left_users) + left_users: Final = seed_users(scenario, SWAP_GROUP) + right_users: Final = seed_users(scenario, SWAP_GROUP) + left: Final = created_team(scenario, create_group(candidate, left_users)) + right: Final = created_team(scenario, create_group(candidate, right_users)) + rounds: Final = tuple( + (left_users, right_users) if round_index % 2 == 0 else (right_users, left_users) + for round_index in range(SWAP_ROUNDS) + ) + for held in rounds: _trade_rosters(candidate, (left, right), held) - _assert_landed(candidate, left, left_users) - _assert_landed(candidate, right, right_users) + assert_landed(candidate, left, left_users) + assert_landed(candidate, right, right_users) def _int_value(rows: Sequence[Mapping[str, JsonValue]], column: str) -> int: @@ -286,6 +241,10 @@ def _sessions_blocked_by(backend_pid: int) -> int: return _int_value(read_rows(SESSIONS_BLOCKED_BY_SQL, (str(backend_pid),)), "waiting") +def _backend_blocked_by(backend_pid: int) -> int: + return _int_value(read_rows(BACKENDS_BLOCKED_BY_SQL, (str(backend_pid),)), "pid") + + def _database_now() -> str: return string_value(read_rows("SELECT now()::text AS now", ())[0]["now"]) @@ -302,36 +261,25 @@ def _roster_ids(team: str) -> tuple[str, ...]: return tuple(string_value(object_value(member)["user_id"]) for member in roster) -def _delete_team_if_present(candidate: Gateway, team: str) -> None: - if read_rows(TEAM_ROW_SQL, (team,)): - _ = candidate.post("/team/delete", {"team_ids": [team]}) - - -def _group_the_test_deletes(candidate: Gateway, scenario: Scenario) -> str: - created: Final = _create_group(candidate, ()) - assert created.status_code == 201, created.text - team: Final = string_value(_body(created)["id"]) - scenario.cleanups.callback(_delete_team_if_present, candidate, team) - return team - - -def test_a_group_deleted_while_its_push_waits_on_a_member_row_answers_404_and_writes_nothing( +def test_a_group_deleted_while_its_push_waits_on_a_member_row_waits_for_the_push_then_removes_every_trace( gateway: Gateway, ) -> None: - with gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=1) as pool: - users: Final = _seed_users(scenario, SMALL_PUSH) - team: Final = _group_the_test_deletes(gateway, scenario) + with gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=2) as pool: + users: Final = seed_users(scenario, SMALL_PUSH) + team: Final = created_team(scenario, create_group(gateway, ())) with _holding_row_lock(LOCK_USER_ROW_SQL, min(users)) as holder_pid: - push: Final = pool.submit(_replace_group, gateway, team, users) + push: Final = pool.submit(replace_group, gateway, team, users) _ = eventually(lambda: _sessions_blocked_by(holder_pid), lambda waiting: waiting >= 1, seconds=30) - deleted: Final = gateway.request("DELETE", f"/scim/v2/Groups/{team}") - assert deleted.status_code == 204, deleted.text + push_pid: Final = _backend_blocked_by(holder_pid) + deleted: Final = pool.submit(delete_group, gateway, team) + _ = eventually(lambda: _sessions_blocked_by(push_pid), lambda waiting: waiting >= 1, seconds=30) assert not push.done(), push.result().text - pushed: Final = push.result() - assert pushed.status_code == 404, pushed.text + assert not deleted.done(), deleted.result().text + assert push.result().status_code == 200, push.result().text + assert deleted.result().status_code == 204, deleted.result().text assert read_rows(TEAM_ROW_SQL, (team,)) == [] - assert _membership_user_ids(team) == frozenset() - assert _users_referencing(team) == frozenset() + assert membership_user_ids(team) == frozenset() + assert users_referencing(team) == frozenset() @contextmanager @@ -361,17 +309,17 @@ def test_a_push_whose_membership_write_fails_answers_5xx_keeps_the_old_roster_an gateway: Gateway, ) -> None: with gateway.scenario() as scenario: - kept: Final = _seed_users(scenario, SMALL_PUSH) - arriving: Final = _seed_users(scenario, SMALL_PUSH) - team: Final = _created_team(scenario, _create_group(gateway, kept)) + kept: Final = seed_users(scenario, SMALL_PUSH) + arriving: Final = seed_users(scenario, SMALL_PUSH) + team: Final = created_team(scenario, create_group(gateway, kept)) replacement: Final = (*kept[:2], *arriving) with _refusing_memberships_for(max(arriving)): - refused: Final = _replace_group(gateway, team, replacement) + refused: Final = replace_group(gateway, team, replacement) assert 500 <= refused.status_code < 600, refused.text - _assert_landed(gateway, team, kept) - retried: Final = _replace_group(gateway, team, replacement) + assert_landed(gateway, team, kept) + retried: Final = replace_group(gateway, team, replacement) assert retried.status_code == 200, retried.text - _assert_landed(gateway, team, replacement) + assert_landed(gateway, team, replacement) def _five_kilobyte_member(scenario: Scenario) -> str: @@ -394,8 +342,8 @@ def test_a_push_with_an_odd_member_value_never_answers_500_and_keeps_the_roster_ gateway: Gateway, shape: str ) -> None: with gateway.scenario() as scenario: - kept: Final = _seed_users(scenario, SMALL_PUSH) - team: Final = _created_team(scenario, _create_group(gateway, kept)) + kept: Final = seed_users(scenario, SMALL_PUSH) + team: Final = created_team(scenario, create_group(gateway, kept)) odd: Final = ODD_MEMBER_VALUES[shape](scenario, kept) pushed: Final = gateway.request( "PUT", @@ -404,14 +352,14 @@ def test_a_push_with_an_odd_member_value_never_answers_500_and_keeps_the_roster_ "schemas": [GROUP_SCHEMA], "id": team, "displayName": f"integration-{uuid.uuid4().hex}", - "members": [*_members(kept), {"value": odd}], + "members": [*members(kept), {"value": odd}], }, ) assert pushed.status_code < 500, pushed.text roster: Final = _roster_ids(team) assert len(roster) == len(frozenset(roster)), roster assert frozenset(kept) <= frozenset(roster), roster - _assert_landed(gateway, team, roster) + assert_landed(gateway, team, roster) if pushed.status_code >= 400: assert frozenset(roster) == frozenset(kept), (pushed.text, roster) @@ -443,15 +391,15 @@ PATCH_SHAPES: Final[Mapping[str, PatchShape]] = { @pytest.mark.parametrize("shape", tuple(PATCH_SHAPES)) def test_patch_member_operation_shapes_land_the_expected_roster(gateway: Gateway, shape: str) -> None: with gateway.scenario() as scenario: - users: Final = _seed_users(scenario, 4) - team: Final = _created_team(scenario, _create_group(gateway, users[:3])) + users: Final = seed_users(scenario, 4) + team: Final = created_team(scenario, create_group(gateway, users[:3])) patched: Final = gateway.request( "PATCH", f"/scim/v2/Groups/{team}", {"schemas": [PATCH_OP_SCHEMA], "Operations": PATCH_SHAPES[shape].operations(users)}, ) assert patched.status_code == 200, patched.text - _assert_landed(gateway, team, PATCH_SHAPES[shape].expected(users)) + assert_landed(gateway, team, PATCH_SHAPES[shape].expected(users)) def _team_key_for(scenario: Scenario, team: str, user_id: str) -> str: @@ -462,7 +410,7 @@ def _team_key_for(scenario: Scenario, team: str, user_id: str) -> str: def _push_outcome(candidate: Gateway, team: str, users: Sequence[str]) -> str: try: - return str(_replace_group(candidate, team, users).status_code) + return str(replace_group(candidate, team, users).status_code) except httpx.TransportError as error: return type(error).__name__ @@ -484,9 +432,9 @@ def test_a_worker_killed_mid_push_leaves_the_old_roster_and_a_retry_lands_the_ne ThreadPoolExecutor(max_workers=1) as pool, ): candidate: Final = owned.gateway - leaving: Final = _seed_users(scenario, CHAOS_GROUP) - arriving: Final = _seed_users(scenario, CHAOS_GROUP) - team: Final = _created_team(scenario, _create_group(candidate, leaving)) + leaving: Final = seed_users(scenario, CHAOS_GROUP) + arriving: Final = seed_users(scenario, CHAOS_GROUP) + team: Final = created_team(scenario, create_group(candidate, leaving)) pinned_key: Final = _team_key_for(scenario, team, max(leaving)) with _holding_row_lock(LOCK_KEY_ROW_SQL, sha256(pinned_key.encode()).hexdigest()) as holder_pid: push: Final = pool.submit(_push_outcome, candidate, team, arriving) @@ -508,11 +456,11 @@ def test_a_worker_killed_mid_push_leaves_the_old_roster_and_a_retry_lands_the_ne {"leaving": len(settled & frozenset(leaving)), "arriving": len(settled & frozenset(arriving))}, ) assert settled in (frozenset(leaving), frozenset(arriving)), sorted(settled) - _assert_landed(candidate, team, sorted(settled)) + assert_landed(candidate, team, sorted(settled)) - retried: Final = _replace_group(candidate, team, arriving) + retried: Final = replace_group(candidate, team, arriving) assert retried.status_code == 200, retried.text - _assert_landed(candidate, team, arriving) + assert_landed(candidate, team, arriving) assert ( read_rows( 'SELECT token FROM "LiteLLM_VerificationToken" WHERE token = %s', diff --git a/tests/integration/management/test_scim_group_delete.py b/tests/integration/management/test_scim_group_delete.py new file mode 100644 index 00000000000..4a4a93f8bd2 --- /dev/null +++ b/tests/integration/management/test_scim_group_delete.py @@ -0,0 +1,431 @@ +"""DELETE /scim/v2/Groups/{id} deletes the group's team the way /team/delete does. + +The members' group entries, their membership rows and the keys the members hold on the team go in one locked +pass whatever the roster size, and the configured admin group's members are demoted in that same pass, so they stay +admins until the group is gone. A member a concurrent group write added while the delete waited on the team lock is +demoted with the rest, and a group write that arrives once the delete holds the lock finds the group gone. The group +is 404 after, on every worker and on the peer proxy, and an unrelated key keeps serving. +A service-account key and a model on the team stay, so re-creating the group with the same id brings them back. +""" + +import os +import re +import uuid +from collections.abc import Iterator +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from functools import partial +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import pytest +import yaml +from integration._support.client import GATEWAY_LIMITS, JSON_OBJECT, Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.scim import ( + GROUP_SCHEMA, + add_to_group, + assert_gone, + assert_landed, + body, + create_group, + created_team, + delete_group, + groups_of, + held_team_lock, + held_user_row, + keys_of, + member_ids, + membership_user_ids, + replace_group, + seed_users, + service_account_key, + team_key, + user_role, + waiters_on_lock_held_by, + waiters_on_rows_locked_by, +) +from pydantic import JsonValue, TypeAdapter + +from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS + +pytestmark: Final = pytest.mark.shares_worker_state + +PROXY_ADMIN: Final = "proxy_admin" +DEMOTED_ROLE: Final = "internal_user_viewer" +MASTER_KEY_USER: Final = "default_user_id" +BURST_GROUPS: Final = 20 +OWNED_PROXY_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120) +DELETED_TEAM_RECORDS_SQL: Final = 'SELECT team_id FROM "LiteLLM_DeletedTeamTable" WHERE team_id = %s' +DELETED_KEY_RECORDS_SQL: Final = 'SELECT token FROM "LiteLLM_DeletedVerificationToken" WHERE token = %s' +TEAM_MODEL_IDS_SQL: Final = """SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_info->>'team_id' = %s""" +BACKENDS_BLOCKED_BY_SQL: Final = "SELECT pid FROM pg_stat_activity WHERE %s::int = ANY(pg_blocking_pids(pid))" +USER_ROLE_SQL: Final = 'SELECT user_role FROM "LiteLLM_UserTable" WHERE user_id = %s' +DELETED_AUDIT_ROWS_SQL: Final = """ +SELECT table_name, action, object_id, changed_by, changed_by_api_key +FROM "LiteLLM_AuditLog" +WHERE object_id IN (%s, %s) AND action = 'deleted' +ORDER BY table_name +""" +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_PIDS: Final = TypeAdapter(tuple[int, ...]) + + +def _hashed(key: str) -> str: + return sha256(key.encode()).hexdigest() + + +def _admin_group_config(tmp_path: Path, admin_alias: str) -> Path: + config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + settings: Final = {**object_value(config["litellm_settings"]), "scim_admin_group": admin_alias} + path: Final = tmp_path / "scim-admin-group.yaml" + path.write_text(yaml.safe_dump({**config, "litellm_settings": settings})) + return path + + +def _roles_of(candidate: Gateway, users: tuple[str, ...]) -> frozenset[str]: + return frozenset(user_role(candidate, user) for user in users) + + +def _backends_blocked_by(holder: int) -> tuple[int, ...]: + return _PIDS.validate_python([row["pid"] for row in read_rows(BACKENDS_BLOCKED_BY_SQL, (str(holder),))]) + + +def _stored_roles(users: tuple[str, ...]) -> frozenset[str]: + return frozenset(string_value(read_rows(USER_ROLE_SQL, (user,))[0]["user_role"]) for user in users) + + +def _models_status(candidate: Gateway, key: str) -> int: + return candidate.request("GET", "/v1/models", key=key).status_code + + +def _team_model_ids(team: str) -> frozenset[str]: + return frozenset(string_value(row["model_id"]) for row in read_rows(TEAM_MODEL_IDS_SQL, (team,))) + + +def _assert_revoked_everywhere(proxies: tuple[Gateway, ...], key: str) -> None: + for candidate in proxies: + eventually(partial(_models_status, candidate, key), lambda code: code == 401, seconds=30) + + +def _deleted_audit_rows(team: str, hashed_key: str) -> list[dict[str, JsonValue]]: + return read_rows(DELETED_AUDIT_ROWS_SQL, (team, hashed_key)) + + +def _worker_startups(log: Path) -> tuple[tuple[int, ...], int]: + text: Final = log.read_text() + return _PIDS.validate_python(_STARTED_WORKER.findall(text)), text.count("Application startup complete.") + + +@contextmanager +def _fresh_connection(candidate: Gateway) -> Iterator[Gateway]: + with httpx.Client( + base_url=candidate.client.base_url, timeout=candidate.client.timeout, trust_env=False, limits=GATEWAY_LIMITS + ) as client: + yield Gateway(client, candidate.key, candidate.upstream_url) + + +def test_deleting_one_group_leaves_the_members_other_group_intact(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + users: Final = seed_users(scenario, 3) + doomed: Final = created_team(scenario, create_group(gateway, users)) + kept: Final = created_team(scenario, create_group(gateway, users)) + kept_key: Final = team_key(gateway, kept, users[0]) + + deleted: Final = delete_group(gateway, doomed) + assert deleted.status_code == 204, deleted.text + + assert gateway.request("GET", f"/scim/v2/Groups/{doomed}").status_code == 404 + assert_landed(gateway, kept, users) + assert groups_of(gateway, users[0]) == frozenset({kept}) + assert keys_of(kept) == frozenset({_hashed(kept_key)}) + assert _models_status(gateway, kept_key) == 200 + + +@pytest.mark.timeout(OWNED_PROXY_TIMEOUT) +def test_deleting_the_admin_group_demotes_its_members_and_writes_an_audit_row_naming_the_scim_caller( + gateway: Gateway, tmp_path: Path +) -> None: + admin_alias: Final = f"integration-admins-{uuid.uuid4().hex}" + with ( + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"], "LITELLM_STORE_AUDIT_LOGS": "true"}, + config=_admin_group_config(tmp_path, admin_alias), + remove_environment=("DATABASE_URL_READ_REPLICA",), + workers=2, + ) as owned, + owned.gateway.scenario() as scenario, + ): + candidate: Final = owned.gateway + users: Final = seed_users(scenario, 3) + team: Final = created_team(scenario, create_group(candidate, users, display_name=admin_alias)) + assert _roles_of(candidate, users) == frozenset({PROXY_ADMIN}) + key: Final = team_key(candidate, team, users[0]) + + deleted: Final = delete_group(candidate, team) + assert deleted.status_code == 204, deleted.text + + assert_gone(candidate, team, users, key) + assert _roles_of(candidate, users) == frozenset({DEMOTED_ROLE}) + rows: Final = eventually(lambda: _deleted_audit_rows(team, _hashed(key)), lambda found: len(found) >= 2, 30) + caller: Final = {"changed_by": MASTER_KEY_USER, "changed_by_api_key": LITELLM_PROXY_MASTER_KEY_ALIAS} + assert rows == [ + {"table_name": "LiteLLM_TeamTable", "action": "deleted", "object_id": team, **caller}, + {"table_name": "LiteLLM_VerificationToken", "action": "deleted", "object_id": _hashed(key), **caller}, + ] + + +@pytest.mark.timeout(OWNED_PROXY_TIMEOUT) +def test_a_member_added_while_the_admin_group_delete_waits_on_the_team_lock_is_demoted_too( + gateway: Gateway, tmp_path: Path +) -> None: + admin_alias: Final = f"integration-admins-{uuid.uuid4().hex}" + with ( + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"]}, + config=_admin_group_config(tmp_path, admin_alias), + remove_environment=("DATABASE_URL_READ_REPLICA",), + workers=2, + client_timeout=60, + ) as owned, + owned.gateway.scenario() as scenario, + ThreadPoolExecutor(max_workers=2) as pool, + ): + candidate: Final = owned.gateway + users: Final = seed_users(scenario, 5) + (late,) = seed_users(scenario, 1) + team: Final = created_team(scenario, create_group(candidate, users, display_name=admin_alias)) + assert _roles_of(candidate, users) == frozenset({PROXY_ADMIN}) + key: Final = team_key(candidate, team, users[0]) + + with held_team_lock(team) as holder: + put: Final = pool.submit(replace_group, candidate, team, (*users, late), display_name=admin_alias) + eventually(lambda: waiters_on_lock_held_by(holder), lambda waiting: waiting == 1, seconds=30) + delete: Final = pool.submit(delete_group, candidate, team) + eventually(lambda: waiters_on_lock_held_by(holder), lambda waiting: waiting == 2, seconds=30) + assert _stored_roles(users) == frozenset({PROXY_ADMIN}) + replaced: Final = put.result() + deleted: Final = delete.result() + + assert replaced.status_code == 200, replaced.text + assert member_ids(body(replaced)) == frozenset((*users, late)) + assert deleted.status_code == 204, deleted.text + assert_gone(candidate, team, (*users, late), key) + assert _roles_of(candidate, (*users, late)) == frozenset({DEMOTED_ROLE}) + + +@pytest.mark.timeout(OWNED_PROXY_TIMEOUT) +def test_a_member_added_while_the_admin_group_delete_holds_the_team_lock_is_refused_and_never_made_admin( + gateway: Gateway, tmp_path: Path +) -> None: + admin_alias: Final = f"integration-admins-{uuid.uuid4().hex}" + with ( + owned_proxy_process( + gateway, + tmp_path, + {"DATABASE_URL": os.environ["DATABASE_URL"]}, + config=_admin_group_config(tmp_path, admin_alias), + remove_environment=("DATABASE_URL_READ_REPLICA",), + workers=2, + client_timeout=60, + ) as owned, + owned.gateway.scenario() as scenario, + ThreadPoolExecutor(max_workers=2) as pool, + ): + candidate: Final = owned.gateway + users: Final = seed_users(scenario, 5) + (late,) = seed_users(scenario, 1) + team: Final = created_team(scenario, create_group(candidate, users, display_name=admin_alias)) + assert _roles_of(candidate, users) == frozenset({PROXY_ADMIN}) + key: Final = team_key(candidate, team, users[0]) + + with held_user_row(users[0]) as holder: + delete: Final = pool.submit(delete_group, candidate, team) + eventually(lambda: waiters_on_rows_locked_by(holder), lambda waiting: waiting == 1, seconds=30) + (deleting,) = _backends_blocked_by(holder) + added: Final = pool.submit(add_to_group, candidate, team, (late,)) + eventually(lambda: _backends_blocked_by(deleting), lambda blocked: len(blocked) == 1, seconds=30) + assert _stored_roles(users) == frozenset({PROXY_ADMIN}) + deleted: Final = delete.result() + + assert deleted.status_code == 204, deleted.text + assert added.result().status_code == 404, added.result().text + assert_gone(candidate, team, users, key) + assert _roles_of(candidate, users) == frozenset({DEMOTED_ROLE}) + assert user_role(candidate, late) != PROXY_ADMIN + + +def test_a_group_delete_revokes_member_keys_and_a_re_created_group_gets_its_service_account_key_and_model_back( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + users: Final = seed_users(scenario, 2) + team: Final = created_team(scenario, create_group(gateway, users)) + member_key: Final = team_key(gateway, team, users[0]) + service_key: Final = service_account_key(gateway, scenario, team) + scenario.model(model_info={"team_id": team}) + team_models: Final = _team_model_ids(team) + assert team_models != frozenset() + + deleted: Final = delete_group(gateway, team) + assert deleted.status_code == 204, deleted.text + + assert gateway.request("GET", f"/scim/v2/Groups/{team}").status_code == 404 + assert membership_user_ids(team) == frozenset() + assert keys_of(team) == frozenset({_hashed(service_key)}) + assert _team_model_ids(team) == team_models + _assert_revoked_everywhere((gateway,), member_key) + + recreated: Final = gateway.request( + "POST", + "/scim/v2/Groups", + {"schemas": [GROUP_SCHEMA], "externalId": team, "displayName": f"integration-{uuid.uuid4().hex}"}, + ) + assert recreated.status_code == 201, recreated.text + assert string_value(body(recreated)["id"]) == team + eventually(partial(_models_status, gateway, service_key), lambda code: code == 200, seconds=90) + assert _models_status(gateway, member_key) == 401 + + +def test_deleting_an_unknown_group_is_404(gateway: Gateway) -> None: + missing: Final = f"missing-{uuid.uuid4().hex}" + response: Final = delete_group(gateway, missing) + assert response.status_code == 404, response.text + assert "error" in body(response), response.text + assert missing in response.text + + +def test_deleting_a_group_twice_is_404_the_second_time(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + users: Final = seed_users(scenario, 2) + team: Final = created_team(scenario, create_group(gateway, users)) + first: Final = delete_group(gateway, team) + assert first.status_code == 204, first.text + second: Final = delete_group(gateway, team) + assert second.status_code == 404, second.text + assert "error" in body(second), second.text + assert gateway.request("GET", f"/scim/v2/Groups/{team}").status_code == 404 + assert membership_user_ids(team) == frozenset() + + +def test_two_deletes_racing_on_one_group_answer_204_then_404_and_record_the_team_once(gateway: Gateway) -> None: + with gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=2) as pool: + users: Final = seed_users(scenario, 2) + team: Final = created_team(scenario, create_group(gateway, users)) + key: Final = team_key(gateway, team, users[0]) + + with held_team_lock(team) as holder: + deletes: Final = (pool.submit(delete_group, gateway, team), pool.submit(delete_group, gateway, team)) + eventually(lambda: waiters_on_lock_held_by(holder), lambda waiting: waiting == 2, seconds=30) + + assert sorted(delete.result().status_code for delete in deletes) == [204, 404] + assert len(read_rows(DELETED_TEAM_RECORDS_SQL, (team,))) == 1 + assert len(read_rows(DELETED_KEY_RECORDS_SQL, (_hashed(key),))) == 1 + assert gateway.request("GET", f"/scim/v2/Groups/{team}").status_code == 404 + + +def test_deleting_a_group_with_a_non_admin_key_is_refused(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + users: Final = seed_users(scenario, 2) + team: Final = created_team(scenario, create_group(gateway, users)) + outsider: Final = scenario.key() + response: Final = delete_group(gateway, team, key=outsider) + assert response.status_code in (401, 403), response.text + assert "error" in body(response), response.text + assert_landed(gateway, team, users) + + +@pytest.mark.parametrize("group_id", ("123", "x" * 5000, "a%2Fb"), ids=("int_like", "five_kb", "encoded_slash")) +def test_deleting_a_group_with_a_hostile_id_is_404(gateway: Gateway, group_id: str) -> None: + response: Final = delete_group(gateway, group_id) + assert response.status_code == 404, response.text + assert isinstance(response.json(), dict), response.text + assert gateway.request("GET", "/health/liveliness").status_code == 200 + + +def test_deleting_an_empty_group_is_204_then_404(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = created_team(scenario, create_group(gateway, ())) + assert member_ids(gateway.get(f"/scim/v2/Groups/{team}")) == frozenset() + deleted: Final = delete_group(gateway, team) + assert deleted.status_code == 204, deleted.text + assert gateway.request("GET", f"/scim/v2/Groups/{team}").status_code == 404 + assert membership_user_ids(team) == frozenset() + + +def test_group_delete_on_one_proxy_revokes_the_team_key_and_the_group_on_the_peer( + gateway: Gateway, peer: Gateway +) -> None: + with gateway.scenario() as scenario: + users: Final = seed_users(scenario, 3) + team: Final = created_team(scenario, create_group(gateway, users)) + key: Final = team_key(gateway, team, users[0]) + assert _models_status(peer, key) == 200 + assert member_ids(peer.get(f"/scim/v2/Groups/{team}")) == frozenset(users) + + deleted: Final = delete_group(gateway, team) + assert deleted.status_code == 204, deleted.text + + assert peer.request("GET", f"/scim/v2/Groups/{team}").status_code == 404 + _assert_revoked_everywhere((peer, gateway), key) + assert_gone(peer, team, users, key) + + +def test_twenty_group_deletes_split_across_two_proxies_all_land(gateway: Gateway, peer: Gateway) -> None: + proxies: Final = (gateway, peer) + with gateway.scenario() as scenario, ThreadPoolExecutor(max_workers=10) as pool: + bystander: Final = scenario.key() + rosters: Final = tuple(seed_users(scenario, 3) for _ in range(BURST_GROUPS)) + teams: Final = tuple(created_team(scenario, create_group(gateway, users)) for users in rosters) + keys: Final = tuple(team_key(gateway, team, users[0]) for team, users in zip(teams, rosters)) + + assigned: Final = tuple(proxies[index % 2] for index in range(BURST_GROUPS)) + responses: Final = tuple(pool.map(delete_group, assigned, teams)) + + for response in responses: + assert response.status_code == 204, response.text + for team, users, key in zip(teams, rosters, keys): + _assert_revoked_everywhere(proxies, key) + assert_gone(gateway, team, users, key) + assert peer.request("GET", f"/scim/v2/Groups/{team}").status_code == 404 + assert _models_status(gateway, bystander) == 200 + assert _models_status(peer, bystander) == 200 + + +@pytest.mark.timeout(OWNED_PROXY_TIMEOUT) +def test_group_delete_after_a_worker_was_killed_still_completes(gateway: Gateway, tmp_path: Path) -> None: + with ( + owned_proxy_process( + gateway, tmp_path, {}, remove_environment=("DATABASE_URL_READ_REPLICA",), workers=2 + ) as owned, + owned.gateway.scenario() as scenario, + ): + candidate: Final = owned.gateway + workers, _ = eventually( + lambda: _worker_startups(owned.log), lambda found: len(found[0]) == 2 and found[1] == 2, seconds=120 + ) + first_users: Final = seed_users(scenario, 3) + second_users: Final = seed_users(scenario, 3) + first: Final = created_team(scenario, create_group(candidate, first_users)) + second: Final = created_team(scenario, create_group(candidate, second_users)) + first_key: Final = team_key(candidate, first, first_users[0]) + second_key: Final = team_key(candidate, second, second_users[0]) + + with _fresh_connection(candidate) as survivor: + with owned.replacing_worker(workers[0]) as kill: + kill.freeze() + kill() + deleted_first: Final = delete_group(survivor, first) + assert deleted_first.status_code == 204, deleted_first.text + assert_gone(survivor, first, first_users, first_key) + + deleted_second: Final = delete_group(survivor, second) + assert deleted_second.status_code == 204, deleted_second.text + assert_gone(survivor, second, second_users, second_key) diff --git a/tests/unit/proxy/hooks/test_key_management_event_hooks.py b/tests/unit/proxy/hooks/test_key_management_event_hooks.py index 7a9d155cd67..8952951351c 100644 --- a/tests/unit/proxy/hooks/test_key_management_event_hooks.py +++ b/tests/unit/proxy/hooks/test_key_management_event_hooks.py @@ -6,6 +6,7 @@ Validates that email and secret manager operations are independent and non-block import asyncio import json +from collections.abc import Sequence from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -715,14 +716,14 @@ async def test_key_deleted_hook_writes_audit_log_for_alias_deletion(): captured: Final[list[LiteLLM_AuditLogs]] = [] - async def capture_audit_log(request_data: LiteLLM_AuditLogs) -> None: - captured.append(request_data) + async def capture_audit_logs(rows: Sequence[LiteLLM_AuditLogs]) -> None: + captured.extend(rows) with ( patch("litellm.store_audit_logs", True), patch( - "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", - new=capture_audit_log, + "litellm.proxy.management_helpers.audit_logs.create_audit_logs_for_updates", + new=capture_audit_logs, ), patch.object( KeyManagementEventHooks, @@ -747,3 +748,34 @@ async def test_key_deleted_hook_writes_audit_log_for_alias_deletion(): assert audit_row.object_id == "hashed" assert audit_row.table_name == LitellmTableNames.KEY_TABLE_NAME assert audit_row.changed_by == "admin" + + +@pytest.mark.asyncio +async def test_deleting_many_keys_writes_every_audit_row_in_one_insert(): + from litellm.proxy._types import LiteLLM_VerificationToken, LitellmTableNames, UserAPIKeyAuth + from litellm.proxy.management_helpers.audit_logs import drain_audit_tasks + + tokens: Final = tuple(f"hashed-{index}" for index in range(50)) + prisma_client: Final = MagicMock() + prisma_client.db.litellm_auditlog.create = AsyncMock() + prisma_client.db.litellm_auditlog.create_many = AsyncMock(return_value=len(tokens)) + + with ( + patch("litellm.store_audit_logs", True), + patch("litellm.proxy.proxy_server.premium_user", True), + patch("litellm.proxy.proxy_server.prisma_client", prisma_client), + ): + KeyManagementEventHooks.create_key_deleted_audit_logs( + keys_being_deleted=[LiteLLM_VerificationToken(token=token) for token in tokens], + user_api_key_dict=UserAPIKeyAuth(user_id="admin", token="callertok"), + ) + await drain_audit_tasks() + + prisma_client.db.litellm_auditlog.create.assert_not_called() + prisma_client.db.litellm_auditlog.create_many.assert_awaited_once() + rows: Final = prisma_client.db.litellm_auditlog.create_many.await_args.kwargs["data"] + assert [row["object_id"] for row in rows] == list(tokens) + assert [json.loads(row["before_value"])["token"] for row in rows] == list(tokens) + assert {(row["table_name"], row["action"], row["changed_by"]) for row in rows} == { + (LitellmTableNames.KEY_TABLE_NAME, "deleted", "admin") + } diff --git a/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py b/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py index cf505e2f98e..6014c9229f6 100644 --- a/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py +++ b/tests/unit/proxy/management_endpoints/scim/test_scim_v2_endpoints.py @@ -3,7 +3,7 @@ import asyncio import json import logging import time -from collections.abc import Callable, Mapping, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from itertools import chain from types import MappingProxyType from typing import Final, assert_never @@ -58,6 +58,7 @@ from litellm.proxy.management_endpoints.scim.scim_v2 import ( update_user, user_api_key_auth, ) +from litellm.proxy.management_endpoints.team_endpoints import DeletedTeams from litellm.proxy.management_helpers.bulk_user_creation import BULK_NEW_USER_CONCURRENCY from litellm.proxy.management_helpers.team_roster_sync import ( MembersMissing, @@ -2831,16 +2832,24 @@ async def test_patch_user_grants_admin_by_team_display_name(mocker, monkeypatch) assert call_args[1]["data"]["user_role"] == LitellmUserRoles.PROXY_ADMIN -def _scim_admin_prisma(mocker, *, user_teams): - """Prisma double and its one transaction, in which the user resolves to user_teams - and teams expose an alias equal to their id. Only the transaction answers reads, - so a role written outside it fails the role-recompute helper tests.""" - user = mocker.MagicMock() - user.user_id = "member-1" - user.teams = user_teams +def _member_row(mocker, user_id: str, teams: Sequence[str]) -> MagicMock: + row = mocker.MagicMock() + row.user_id = user_id + row.teams = list(teams) + return row + + +def _scim_admin_prisma(mocker, *, user_teams, teammates: Sequence[tuple[str, Sequence[str]]] = ()): + """Prisma double and its one transaction, in which the user resolves to user_teams, + ``teammates`` are further (user_id, teams) rows, and teams expose an alias equal to + their id. Only the transaction answers reads, so a role written outside it fails the + role-recompute helper tests.""" + user = _member_row(mocker, "member-1", user_teams) + members = (user, *(_member_row(mocker, user_id, teams) for user_id, teams in teammates)) def _team_find_unique(where): team = mocker.MagicMock() + team.team_id = where["team_id"] team.team_alias = where["team_id"] return team @@ -2848,7 +2857,7 @@ def _scim_admin_prisma(mocker, *, user_teams): return tuple(_team_find_unique({"team_id": team_id}) for team_id in where["team_id"]["in"]) def _users_find_many(where): - return (user,) if user.user_id in where["user_id"]["in"] else () + return tuple(member for member in members if member.user_id in where["user_id"]["in"]) tx = mocker.MagicMock() tx.query_raw = AsyncMock(return_value=[]) @@ -2904,6 +2913,29 @@ async def test_recompute_scim_member_roles_grants_when_in_admin_group(mocker, mo tx.execute_raw.assert_awaited_once_with(ANY, LitellmUserRoles.PROXY_ADMIN, ["member-1"]) +@pytest.mark.asyncio +async def test_recompute_scim_member_roles_stops_serving_a_demoted_members_cached_admin_role(mocker, monkeypatch): + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.proxy_server import proxy_config + + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": "litellm-admins"}} + + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + cache = UserApiKeyCache() + await cache.async_set_cache( + key="member-1", value={"user_id": "member-1", "user_role": LitellmUserRoles.PROXY_ADMIN.value} + ) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + prisma, tx = _scim_admin_prisma(mocker, user_teams=["engineering"]) + + await _recompute_scim_member_roles(prisma, ["member-1"]) + + tx.execute_raw.assert_awaited_once_with(ANY, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, ["member-1"]) + assert await cache.async_get_cache(key="member-1") is None + + @pytest.mark.asyncio async def test_recompute_scim_member_roles_noop_when_admin_group_unset(mocker, monkeypatch): """With scim_admin_group unset the recompute helper must not touch any role, @@ -3043,44 +3075,128 @@ async def test_patch_group_recomputes_roles_for_changed_members(mocker): assert set(recompute_mock.call_args[0][1]) == {"user1"} -@pytest.mark.asyncio -async def test_delete_group_recomputes_roles_for_members(mocker): - """DELETE /Groups must recompute the global role for the team's members, so - deleting the admin group demotes everyone who was only admin through it.""" - from litellm.proxy._types import Member +_SCIM_CALLER: Final = UserAPIKeyAuth(api_key="hashed-scim-token", user_id="idp-service-user") +_ADMIN_GROUP: Final = "litellm-admins" +_DEMOTED: Final = call(ANY, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, ["member-1"]) - existing_team = mocker.MagicMock() - existing_team.members_with_roles = [ - Member(user_id="user1", role="user"), - Member(user_id="user2", role="user"), - ] - member = mocker.MagicMock() - member.teams = ["test-team-123"] +def _group_delete_mocks( + mocker: MockerFixture, + monkeypatch: pytest.MonkeyPatch, + team: LiteLLM_TeamTable | None, + *, + teammates: Sequence[tuple[str, Sequence[str]]] = (), +) -> tuple[MagicMock, MagicMock, AsyncMock]: + """Prisma and its transaction, whose users are left in no team once the delete detached + them and whose group lookup answers ``team``, plus the patched team delete, which hands + its transaction and the roster it read under the lock, member-1 and every teammate, to + the caller's write.""" + from litellm.proxy.proxy_server import proxy_config - mock_prisma_client = mocker.MagicMock() - mock_prisma_client.db = mocker.MagicMock() - mock_prisma_client.db.litellm_teamtable = mocker.MagicMock() - mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=existing_team) - mock_prisma_client.db.litellm_teamtable.delete = AsyncMock() - mock_prisma_client.db.litellm_usertable = mocker.MagicMock() - mock_prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=member) - mock_prisma_client.db.litellm_usertable.find_many = AsyncMock(return_value=()) - mock_prisma_client.db.litellm_usertable.update = AsyncMock() + async def mock_get_config(): + return {"litellm_settings": {"scim_admin_group": _ADMIN_GROUP}} + monkeypatch.setattr(proxy_config, "get_config", mock_get_config) + monkeypatch.setattr("litellm.default_internal_user_params", None, raising=False) + prisma, tx = _scim_admin_prisma(mocker, user_teams=[], teammates=teammates) + prisma.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) mocker.patch( "litellm.proxy.management_endpoints.scim.scim_v2._get_prisma_client_or_raise_exception", - AsyncMock(return_value=mock_prisma_client), + AsyncMock(return_value=prisma), ) - recompute_mock = mocker.patch( - "litellm.proxy.management_endpoints.scim.scim_v2._recompute_scim_member_roles", - AsyncMock(), + roster: Final = ("member-1", *(user_id for user_id, _ in teammates)) + + async def delete_rows( + *, before_commit: Callable[[object, tuple[str, ...]], Awaitable[None]] | None = None, **_: object + ) -> DeletedTeams: + if before_commit is not None: + await before_commit(tx, roster) + return DeletedTeams(team_ids=(_ADMIN_GROUP,)) + + delete_team_mock: Final = mocker.patch( + "litellm.proxy.management_endpoints.scim.scim_v2.delete_validated_team_rows", + AsyncMock(side_effect=delete_rows), ) + return prisma, tx, delete_team_mock - await delete_group(group_id="test-team-123") - recompute_mock.assert_awaited_once() - assert list(recompute_mock.call_args[0][1]) == ["user1", "user2"] +def _admin_group_team() -> LiteLLM_TeamTable: + return LiteLLM_TeamTable(team_id=_ADMIN_GROUP, members_with_roles=[Member(user_id="member-1", role="user")]) + + +@pytest.mark.asyncio +async def test_delete_group_demotes_a_member_who_was_admin_only_through_the_deleted_group( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +): + """DELETE /Groups answers 204 and writes the non-admin role for a member who was + admin only through the deleted group, once, inside the team delete.""" + _, tx, _ = _group_delete_mocks(mocker, monkeypatch, _admin_group_team()) + + response: Final = await delete_group(group_id=_ADMIN_GROUP, user_api_key_dict=_SCIM_CALLER) + + assert response.status_code == 204 + assert tx.execute_raw.await_args_list == [_DEMOTED] + + +@pytest.mark.asyncio +async def test_delete_group_demotes_every_member_on_the_roster_the_team_delete_read_under_the_lock( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +): + """A member a concurrent PUT added to the admin group after this delete read the + group is on the roster the team delete read under the lock, so the one role write + covers that late member too.""" + _, tx, _ = _group_delete_mocks(mocker, monkeypatch, _admin_group_team(), teammates=(("member-2", ()),)) + + response: Final = await delete_group(group_id=_ADMIN_GROUP, user_api_key_dict=_SCIM_CALLER) + + assert response.status_code == 204 + assert tx.execute_raw.await_args_list == [ + call(ANY, LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, ["member-1", "member-2"]), + ] + + +@pytest.mark.asyncio +async def test_delete_group_fails_when_the_role_writes_fail(mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch): + """The role writes run on the team delete's transaction, so when they fail the + request fails with them and the identity provider's retry finds the group.""" + _, tx, _ = _group_delete_mocks(mocker, monkeypatch, _admin_group_team()) + tx.execute_raw.side_effect = RuntimeError("database unavailable") + + with pytest.raises(ProxyException) as raised: + await delete_group(group_id=_ADMIN_GROUP, user_api_key_dict=_SCIM_CALLER) + + assert raised.value.code == "500" + + +@pytest.mark.asyncio +async def test_delete_group_answers_404_when_a_concurrent_delete_removed_the_team_first( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +): + """A concurrent delete that removed the group while this one waited on the team lock + leaves the team delete nothing to delete or report, so this request answers 404, the + same as a group that never existed.""" + _, _, delete_team_mock = _group_delete_mocks(mocker, monkeypatch, _admin_group_team()) + delete_team_mock.side_effect = None + delete_team_mock.return_value = DeletedTeams(team_ids=()) + + with pytest.raises(ProxyException) as raised: + await delete_group(group_id=_ADMIN_GROUP, user_api_key_dict=_SCIM_CALLER) + + assert raised.value.code == "404" + + +@pytest.mark.asyncio +async def test_delete_group_of_an_unknown_group_answers_404_without_deleting_or_touching_roles( + mocker: MockerFixture, monkeypatch: pytest.MonkeyPatch +): + _, tx, delete_team_mock = _group_delete_mocks(mocker, monkeypatch, None) + + with pytest.raises(ProxyException) as raised: + await delete_group(group_id="missing-team", user_api_key_dict=_SCIM_CALLER) + + assert raised.value.code == "404" + delete_team_mock.assert_not_awaited() + tx.execute_raw.assert_not_called() @pytest.mark.asyncio @@ -3121,12 +3237,10 @@ async def test_delete_group_is_refused_while_the_team_holds_a_ptu_share(mocker, monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", shared_router) with pytest.raises(ProxyException) as refused: - await delete_group(group_id="test-team-123") + await delete_group(group_id="test-team-123", user_api_key_dict=_SCIM_CALLER) assert refused.value.code == "400" assert "test-team-123" in str(refused.value.message) and "shared-ptu" in str(refused.value.message) - mock_prisma_client.db.litellm_teamtable.delete.assert_not_called() - mock_prisma_client.db.litellm_usertable.update.assert_not_called() @pytest.mark.asyncio diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index 037ccd26236..cb8a73d581b 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -5123,11 +5123,11 @@ async def test_delete_user_writes_deleted_audit_log_for_user_keys(mocker): mocker.patch("litellm.store_audit_logs", True) captured: Final[list] = [] - async def _capture(request_data): - captured.append(request_data) + async def _capture(rows): + captured.extend(rows) mocker.patch( - "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + "litellm.proxy.management_helpers.audit_logs.create_audit_logs_for_updates", new=_capture, ) diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 5063ae0b357..37028558af3 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -29,6 +29,7 @@ from litellm.proxy._types import ( LiteLLM_TeamTable, LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, + LiteLLM_VerificationToken, LitellmTableNames, LitellmUserRoles, Member, @@ -50,7 +51,9 @@ from litellm.proxy.management_endpoints.team_admin_field_permissions import ( SUPPORTED_TEAM_ADMIN_PERMISSIONS, ) from litellm.proxy.management_endpoints.team_endpoints import ( + _LOCK_DELETED_TEAM_MEMBERS_SQL, _STRIP_DELETED_TEAMS_FROM_USERS_SQL, + DeletedTeams, GetTeamMemberPermissionsResponse, UpdateTeamMemberPermissionsRequest, _build_team_list_where_conditions, @@ -63,6 +66,8 @@ from litellm.proxy.management_endpoints.team_endpoints import ( _validate_team_member_reset_spend_value, aggregated_date_range_error, delete_team, + delete_validated_team_rows, + delete_validated_teams, list_available_teams, new_team, reset_team_member_budget_fn, @@ -212,21 +217,50 @@ def _wire_member_delete_tx(prisma_client): return tx -def _wire_team_delete_tx(prisma_client): - """`/team/delete` deletes the team rows and runs its post-delete reference sweep under - every team's advisory lock in one transaction, so a mocked client has to hand its own - table mocks (and db-level execute_raw) back out of `tx()` for existing per-table - assertions on `prisma_client.db.*` to keep seeing those calls.""" +def _wire_team_delete_tx( + prisma_client, + *, + present_team_ids=None, + locked_teams: tuple[LiteLLM_TeamTable, ...] = (), + team_keys: tuple[LiteLLM_VerificationToken, ...] = (), +): + """`/team/delete` records, deletes and sweeps the team rows under every team's advisory + lock in one transaction, so a mocked client has to hand its own table mocks back out of + `tx()` for existing per-table assertions on `prisma_client.db.*` to keep seeing those + calls. Under the lock every requested team still exists unless `present_team_ids` names + the ones left, and reads as the matching `locked_teams` entry or a bare row. The keys the + lock finds on those teams are `team_keys`. The locked sweep reads the users it detaches + through a `query_raw` whose rows name them and their teams, so the default answers none.""" + locked_by_id = {team.team_id: team for team in locked_teams} + + async def teams_still_there(where: dict[str, dict[str, list[str]]]) -> list[LiteLLM_TeamTable]: + return [ + locked_by_id.get(team_id, LiteLLM_TeamTable(team_id=team_id)) + for team_id in where["team_id"]["in"] + if present_team_ids is None or team_id in present_team_ids + ] + + async def keys_on_teams(where: dict[str, dict[str, list[str]]]) -> list[LiteLLM_VerificationToken]: + return [key for key in team_keys if key.team_id in where["team_id"]["in"]] + tx = SimpleNamespace( - litellm_teamtable=prisma_client.db.litellm_teamtable, + litellm_teamtable=SimpleNamespace( + find_many=teams_still_there, + delete_many=prisma_client.db.litellm_teamtable.delete_many, + ), + litellm_verificationtoken=SimpleNamespace(find_many=keys_on_teams, delete_many=AsyncMock(return_value=0)), + litellm_deletedverificationtoken=SimpleNamespace(create_many=AsyncMock()), + litellm_jwtkeymapping=SimpleNamespace(find_many=AsyncMock(return_value=[])), litellm_teammembership=prisma_client.db.litellm_teammembership, + litellm_deletedteamtable=prisma_client.db.litellm_deletedteamtable, query_raw=AsyncMock(return_value=[]), - execute_raw=prisma_client.db.execute_raw, + execute_raw=AsyncMock(return_value=0), ) tx_cm = MagicMock() tx_cm.__aenter__ = AsyncMock(return_value=tx) tx_cm.__aexit__ = AsyncMock(return_value=None) prisma_client.tx = MagicMock(return_value=tx_cm) + return tx # Mock prisma_client @@ -5489,13 +5523,13 @@ async def test_team_member_delete_writes_deleted_audit_log_for_member_keys( captured: Final[list] = [] - async def _capture(request_data): - captured.append(request_data) + async def _capture(rows): + captured.extend(rows) with ( patch("litellm.store_audit_logs", True), patch( - "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + "litellm.proxy.management_helpers.audit_logs.create_audit_logs_for_updates", new=_capture, ), ): @@ -5564,12 +5598,12 @@ async def test_delete_team_writes_deleted_audit_log_for_team_keys( ) captured: Final[list] = [] - async def _capture(request_data): - captured.append(request_data) + async def _capture(rows): + captured.extend(rows) monkeypatch.setattr("litellm.store_audit_logs", True) monkeypatch.setattr( - "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + "litellm.proxy.management_helpers.audit_logs.create_audit_logs_for_updates", _capture, ) await delete_team( @@ -9050,7 +9084,7 @@ async def test_save_deleted_team_records(): }, ] - await _save_deleted_team_records(records=records, prisma_client=mock_prisma_client) + await _save_deleted_team_records(records=records, deleted_teams=mock_prisma_client.db.litellm_deletedteamtable) mock_create_many.assert_called_once_with(data=records) @@ -9061,7 +9095,7 @@ async def test_save_deleted_team_records_empty_list(): mock_create_many = AsyncMock() mock_prisma_client.db.litellm_deletedteamtable.create_many = mock_create_many - await _save_deleted_team_records(records=[], prisma_client=mock_prisma_client) + await _save_deleted_team_records(records=[], deleted_teams=mock_prisma_client.db.litellm_deletedteamtable) mock_create_many.assert_not_called() @@ -9091,7 +9125,7 @@ async def test_persist_deleted_team_records(): await _persist_deleted_team_records( teams=[team], - prisma_client=mock_prisma_client, + deleted_teams=mock_prisma_client.db.litellm_deletedteamtable, user_api_key_dict=user_api_key_dict, litellm_changed_by="admin-user", ) @@ -9295,7 +9329,7 @@ async def test_delete_team_sweeps_references_outside_members_with_roles( # The locked delete-and-sweep transaction /team/member_add serializes against, kept # separate from mock_tx above (the BYOK-model-cleanup transaction, unrelated to this lock). - _wire_team_delete_tx(mock_prisma_client) + _wire_team_delete_tx(mock_prisma_client, locked_teams=(doomed_team,)) mock_lock_tx = mock_prisma_client.tx.return_value.__aenter__.return_value fresh_cache = UserApiKeyCache() @@ -9325,24 +9359,23 @@ async def test_delete_team_sweeps_references_outside_members_with_roles( litellm_changed_by="admin-user", ) - assert mock_execute_raw.await_args_list == [ - call(_STRIP_DELETED_TEAMS_FROM_USERS_SQL, ["team-doomed"]), - call(_STRIP_DELETED_TEAMS_FROM_USERS_SQL, ["team-doomed"]), + assert mock_execute_raw.await_args_list == [], ( + "no sweep runs outside the lock: one there strips a member a concurrent roster write added " + "after the caller read the team, before the locked sweep can report them" + ) + + # one pass for the membership rows, under the lock alongside the delete + assert mock_membership_delete_many.await_args_list == [call(where={"team_id": {"in": ("team-doomed",)}})] + + assert mock_lock_tx.query_raw.await_args_list == [ + call(TEAM_ADVISORY_LOCK_SQL, "team-doomed"), + call(_LOCK_DELETED_TEAM_MEMBERS_SQL, ["team-doomed"]), ], ( - "the unlocked sweep must run once to catch pre-existing drift, and the locked sweep " - "(alongside the delete, under the same advisory lock member_add takes) must run again " - "so a member_add that wrote its reference just before losing the lock is still reaped" - ) - - # same two passes for the membership rows, the second under the lock alongside the delete - assert mock_membership_delete_many.await_args_list == [ - call(where={"team_id": {"in": ("team-doomed",)}}), - call(where={"team_id": {"in": ("team-doomed",)}}), - ] - - assert mock_lock_tx.query_raw.await_args_list == [call(TEAM_ADVISORY_LOCK_SQL, "team-doomed")], ( - "the advisory lock must be acquired before the team row is deleted" + "the advisory lock must be acquired before the team row is deleted, and the user sweep " + "runs under it (the same lock member_add takes) so a member_add that wrote its " + "reference just before losing the lock is still reaped" ) + assert mock_lock_tx.execute_raw.await_args_list == [call(_STRIP_DELETED_TEAMS_FROM_USERS_SQL, ["team-doomed"])] assert fresh_cache.get_cache(key="team_id:team-doomed") is None assert fresh_cache.get_cache(key="team_alias:doomed-team") is None @@ -9417,7 +9450,7 @@ async def test_delete_team_evicts_member_caches_with_one_transaction( mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) mock_tx_cm.__aexit__ = AsyncMock(return_value=False) mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) - _wire_team_delete_tx(mock_prisma_client) + _wire_team_delete_tx(mock_prisma_client, locked_teams=(team,)) fresh_cache = UserApiKeyCache() for user_id in member_user_ids: @@ -9461,6 +9494,234 @@ async def test_delete_team_evicts_member_caches_with_one_transaction( ) +@pytest.mark.asyncio +async def test_delete_validated_team_rows_records_the_roster_the_lock_read_and_writes_its_users_before_commit( + monkeypatch, + disable_audit_logging_for_mocked_team, +): + """ + A roster write that lands between the caller's read of the team and the locked delete adds + a member the caller never saw, and a user row can reference the team without a roster entry. + The deleted-team record and the caller's write, which runs on the delete's transaction before + it commits, both take the roster read under the lock, so they name the late member and not the + stray user, while both lose their cached user object. + """ + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + read_by_caller = LiteLLM_TeamTable( + team_id="team-doomed", + team_alias="doomed-team", + members_with_roles=[Member(user_id="member-1", role="user")], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + read_under_lock = read_by_caller.model_copy( + update={ + "members_with_roles": [ + Member(user_id="member-1", role="user"), + Member(user_id="late-member", role="user"), + ] + } + ) + + mock_prisma_client = AsyncMock() + mock_prisma_client.delete_data = AsyncMock(return_value={"deleted_keys": 0}) + mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.execute_raw = AsyncMock() + mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() + + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx_cm = MagicMock() + mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) + mock_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) + tx = _wire_team_delete_tx(mock_prisma_client, locked_teams=(read_under_lock,)) + tx.query_raw = AsyncMock( + side_effect=[ + [], + [ + {"user_id": "member-1", "teams": ["team-kept", "team-doomed"]}, + {"user_id": "late-member", "teams": ["team-doomed"]}, + {"user_id": "stray-user", "teams": ["team-doomed"]}, + ], + ] + ) + + fresh_cache = UserApiKeyCache() + for user_id in ("member-1", "late-member", "stray-user", "bystander-user"): + fresh_cache.set_cache(key=user_id, value=UserAPIKeyAuth(user_id=user_id)) + + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", fresh_cache) + monkeypatch.setattr("litellm.proxy.proxy_server.create_audit_log_for_update", AsyncMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + async def write_before_commit(_tx: object, _user_ids: tuple[str, ...]) -> None: + mock_prisma_client.tx.return_value.__aexit__.assert_not_awaited() + + before_commit = AsyncMock(side_effect=write_before_commit) + + deleted = await delete_validated_team_rows( + teams=(read_by_caller,), + prisma_client=mock_prisma_client, + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-admin", + user_role=LitellmUserRoles.PROXY_ADMIN.value, + ), + litellm_changed_by="admin-user", + before_commit=before_commit, + ) + + assert deleted == DeletedTeams(team_ids=("team-doomed",)) + before_commit.assert_awaited_once_with(tx, ("member-1", "late-member")) + (written,) = mock_prisma_client.db.litellm_deletedteamtable.create_many.await_args_list + (record,) = written.kwargs["data"] + assert [member["user_id"] for member in json.loads(record["members_with_roles"])] == ["member-1", "late-member"] + for user_id in ("member-1", "late-member", "stray-user"): + assert fresh_cache.get_cache(key=user_id) is None, user_id + assert fresh_cache.get_cache(key="bystander-user") is not None + + +@pytest.mark.asyncio +async def test_delete_validated_team_rows_deletes_only_the_keys_its_rostered_members_hold_on_the_team( + monkeypatch, + disable_audit_logging_for_mocked_team, +): + """ + The SCIM group delete removes the team the way a SCIM member removal removes each member: + the keys rostered members hold on the team are tombstoned and deleted, and a key with no + user or held by someone off the roster stays. + """ + team = LiteLLM_TeamTable( + team_id="team-doomed", + members_with_roles=[Member(user_id="member-1", role="user")], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + team_keys = tuple( + LiteLLM_VerificationToken(token=token, team_id="team-doomed", user_id=user_id) + for token, user_id in (("hashed-member", "member-1"), ("hashed-service-account", None), ("hashed-outsider", "admin-made")) + ) + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() + mock_access_group_tx_cm = MagicMock() + mock_access_group_tx_cm.__aenter__ = AsyncMock(return_value=AsyncMock()) + mock_access_group_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma_client.db.tx = MagicMock(return_value=mock_access_group_tx_cm) + tx = _wire_team_delete_tx(mock_prisma_client, locked_teams=(team,), team_keys=team_keys) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + await delete_validated_team_rows( + teams=(team,), + prisma_client=mock_prisma_client, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="hashed-scim-token"), + litellm_changed_by=None, + ) + + (tombstones,) = tx.litellm_deletedverificationtoken.create_many.await_args_list + assert [record["token"] for record in tombstones.kwargs["data"]] == ["hashed-member"] + (deleted_keys,) = tx.litellm_verificationtoken.delete_many.await_args_list + assert deleted_keys.kwargs["where"] == {"token": {"in": ["hashed-member"]}} + + +@pytest.mark.asyncio +async def test_delete_validated_teams_writes_the_deleted_audit_row_only_for_a_team_still_there_under_the_lock( + monkeypatch: pytest.MonkeyPatch, +): + """Two deletes racing on one team both read it before the lock, and only the one that still + finds it under the lock deletes it, so only that one writes its deleted audit row.""" + audit_logger = _wire_audit_log_callback(monkeypatch) + teams = tuple( + LiteLLM_TeamTable(team_id=team_id, metadata={}, model_max_budget={}, model_spend={}) + for team_id in ("team-raced", "team-kept-for-us") + ) + mock_prisma = AsyncMock() + mock_prisma.delete_data = AsyncMock(return_value={"deleted_keys": 0}) + mock_prisma.db.litellm_deletedteamtable.create_many = AsyncMock() + mock_prisma.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma.db.litellm_auditlog.create = AsyncMock() + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx_cm = MagicMock() + mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) + mock_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma.db.tx = MagicMock(return_value=mock_tx_cm) + _wire_team_delete_tx(mock_prisma, present_team_ids={"team-kept-for-us"}) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + await delete_validated_teams( + teams=teams, + prisma_client=mock_prisma, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin-1", api_key="sk-a"), + litellm_changed_by=None, + ) + await _settle_audit_log_tasks() + + assert [ + (p["object_id"], p["action"]) + for p in audit_logger.payloads + if p["table_name"] == LitellmTableNames.TEAM_TABLE_NAME + ] == [("team-kept-for-us", "deleted")] + + +@pytest.mark.asyncio +async def test_delete_validated_teams_records_and_reports_only_the_teams_still_there_under_the_lock( + monkeypatch, + disable_audit_logging_for_mocked_team, +): + """ + A concurrent delete that removed one of the teams while this one waited on the lock + already wrote that team's deleted-team record, so this delete writes records only for + the teams it still finds under the lock and reports only those as deleted. + """ + teams = tuple( + LiteLLM_TeamTable( + team_id=team_id, + team_alias=team_id, + members_with_roles=[], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + for team_id in ("team-raced", "team-kept-for-us") + ) + mock_prisma_client = AsyncMock() + mock_prisma_client.delete_data = AsyncMock(return_value={"deleted_keys": 0}) + mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) + mock_prisma_client.db.execute_raw = AsyncMock() + mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx_cm = MagicMock() + mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) + mock_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) + _wire_team_delete_tx(mock_prisma_client, present_team_ids={"team-kept-for-us"}) + monkeypatch.setattr("litellm.proxy.proxy_server.create_audit_log_for_update", AsyncMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + deleted = await delete_validated_teams( + teams=teams, + prisma_client=mock_prisma_client, + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", + api_key="sk-admin", + user_role=LitellmUserRoles.PROXY_ADMIN.value, + ), + litellm_changed_by=None, + ) + + assert deleted.team_ids == ("team-kept-for-us",) + (written,) = mock_prisma_client.db.litellm_deletedteamtable.create_many.await_args_list + assert [record["team_id"] for record in written.kwargs["data"]] == ["team-kept-for-us"] + + @pytest.mark.asyncio async def test_delete_team_evicts_the_auth_cache_of_the_keys_it_deletes( monkeypatch, @@ -9558,8 +9819,7 @@ async def test_delete_team_failing_locked_sweep_rolls_back_the_delete_and_leaves mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock() mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[]) mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() - # the unlocked pre-delete sweep succeeds, the locked post-delete sweep blows up - mock_prisma_client.db.execute_raw = AsyncMock(side_effect=[None, ConnectionError("db went away")]) + mock_prisma_client.db.execute_raw = AsyncMock() mock_tx = AsyncMock() mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) @@ -9568,6 +9828,10 @@ async def test_delete_team_failing_locked_sweep_rolls_back_the_delete_and_leaves mock_tx_cm.__aexit__ = AsyncMock(return_value=False) mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) _wire_team_delete_tx(mock_prisma_client) + # the advisory lock is taken, then the locked post-delete user sweep blows up + mock_prisma_client.tx.return_value.__aenter__.return_value.query_raw = AsyncMock( + side_effect=[[], ConnectionError("db went away")] + ) fresh_cache = UserApiKeyCache() cached_obj = LiteLLM_TeamTableCachedObj(team_id="team-doomed", team_alias="doomed-team") @@ -9607,21 +9871,26 @@ class _BroadcastPipeline: return self async def execute(self) -> None: - for message in self._messages: - await self._redis.publish("", message) + await self._redis.send(tuple(self._messages)) class _BroadcastRecordingRedis: namespace = None - def __init__(self) -> None: + def __init__(self, round_trip_seconds: float = 0.0) -> None: self.broadcast_keys: list[str] = [] + self._round_trip_seconds = round_trip_seconds def init_pubsub_client(self) -> "_BroadcastRecordingRedis": return self + async def send(self, messages: Sequence[str]) -> None: + if self._round_trip_seconds: + await asyncio.sleep(self._round_trip_seconds) + self.broadcast_keys.extend(json.loads(message)["cache_key"] for message in messages) + async def publish(self, channel: str, message: str) -> int: - self.broadcast_keys.append(json.loads(message)["cache_key"]) + await self.send((message,)) return 1 def pipeline(self, transaction: bool = True) -> _BroadcastPipeline: @@ -9667,7 +9936,7 @@ async def test_delete_team_broadcasts_cache_invalidation_to_other_workers( mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) mock_tx_cm.__aexit__ = AsyncMock(return_value=False) mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) - _wire_team_delete_tx(mock_prisma_client) + _wire_team_delete_tx(mock_prisma_client, locked_teams=(team,)) import litellm.proxy.common_utils.auth_cache_invalidation_pubsub as pubsub_module @@ -9697,6 +9966,44 @@ async def test_delete_team_broadcasts_cache_invalidation_to_other_workers( assert redis.broadcast_keys == ["hashed-doomed-key", "team_id:team-doomed", "team_alias:doomed-team"] +@pytest.mark.asyncio +async def test_deleting_a_500_member_team_reaches_every_member_cache_on_other_workers( + monkeypatch: pytest.MonkeyPatch, +): + from itertools import chain + + import litellm.proxy.common_utils.auth_cache_invalidation_pubsub as pubsub_module + from litellm.proxy.common_utils.user_api_key_cache import ( + UserApiKeyCache, + team_membership_auth_cache_key, + team_membership_reservation_cache_key, + ) + from litellm.proxy.management_endpoints.team_endpoints import _evict_deleted_team_member_cache + + redis: Final = _BroadcastRecordingRedis(round_trip_seconds=0.001) + monkeypatch.setattr(pubsub_module, "coordination_redis_cache", lambda: redis) + monkeypatch.setattr(pubsub_module, "_pending_publishes", set()) + monkeypatch.setattr(pubsub_module, "_in_flight_publishes", asyncio.Semaphore(16)) + members: Final = tuple(f"member-{index:03d}" for index in range(500)) + + await _evict_deleted_team_member_cache( + team_id="team-large", member_user_ids=members, user_api_key_cache=UserApiKeyCache() + ) + await asyncio.gather(*pubsub_module._pending_publishes) # pyright: ignore[reportPrivateUsage] # drain module-level tasks + + expected: Final = frozenset( + chain.from_iterable( + ( + member, + team_membership_auth_cache_key(team_id="team-large", user_id=member), + team_membership_reservation_cache_key(user_id=member, team_id="team-large"), + ) + for member in members + ) + ) + assert expected - frozenset(redis.broadcast_keys) == frozenset() + + @pytest.mark.asyncio async def test_delete_team_survives_a_failing_cache_backend( monkeypatch, @@ -15783,6 +16090,8 @@ async def test_new_team_and_delete_team_both_drive_the_mirror( prisma.delete_data = AsyncMock(return_value=[team_row]) prisma.db.execute_raw = AsyncMock(return_value=0) prisma.db.litellm_teammembership.delete_many = AsyncMock(return_value=0) + prisma.db.litellm_teamtable.delete_many = AsyncMock(return_value=1) + _wire_team_delete_tx(prisma, locked_teams=(team_row,)) await delete_team( data=DeleteTeamRequest(team_ids=["team-gone"]), diff --git a/tests/unit/proxy/management_helpers/test_bulk_user_deletion.py b/tests/unit/proxy/management_helpers/test_bulk_user_deletion.py index 74b4a3caf0a..e212a433393 100644 --- a/tests/unit/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/unit/proxy/management_helpers/test_bulk_user_deletion.py @@ -1007,12 +1007,12 @@ async def test_bulk_delete_writes_deleted_audit_log_for_deleted_keys(mocker): captured: Final[list] = [] - async def _capture(request_data): - captured.append(request_data) + async def _capture(rows): + captured.extend(rows) mocker.patch("litellm.store_audit_logs", True) mocker.patch( - "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + "litellm.proxy.management_helpers.audit_logs.create_audit_logs_for_updates", new=_capture, ) await _delete(prisma, ["u1"]) @@ -1041,12 +1041,12 @@ async def test_bulk_member_delete_writes_deleted_audit_log_for_removed_team_keys captured: Final[list] = [] - async def _capture(request_data): - captured.append(request_data) + async def _capture(rows): + captured.extend(rows) mocker.patch("litellm.store_audit_logs", True) mocker.patch( - "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + "litellm.proxy.management_helpers.audit_logs.create_audit_logs_for_updates", new=_capture, ) await _remove(prisma, "t1", [{"user_id": "u1"}]) @@ -1076,7 +1076,7 @@ async def test_bulk_delete_skips_the_key_audit_log_when_the_tx_rolls_back(mocker mocker.patch("litellm.store_audit_logs", True) mock_audit_write = mocker.patch( - "litellm.proxy.management_helpers.audit_logs.create_audit_log_for_update", + "litellm.proxy.management_helpers.audit_logs.create_audit_logs_for_updates", new=mocker.AsyncMock(), ) results = await _delete(prisma, ["u1", "u2"], cache=cache) From 1e1770aa48083a9a267c910a12250d305d0604d0 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 18:19:28 -0700 Subject: [PATCH 036/235] test(perf): emit event_loop_stalls hold counts as perf_ JUnit properties (#47706) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/setup/event_loop.py | 26 +++++++++++++ tests/e2e/perf/test_perf_cases.py | 5 ++- .../e2e_harness/perf/setup/test_event_loop.py | 38 ++++++++++++++++++- 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/tests/e2e/perf/setup/event_loop.py b/tests/e2e/perf/setup/event_loop.py index dee45c374d9..1c582e24fd8 100644 --- a/tests/e2e/perf/setup/event_loop.py +++ b/tests/e2e/perf/setup/event_loop.py @@ -14,6 +14,7 @@ a hold that such a pass accounts for is reported under `garbage collection` inst from __future__ import annotations +import json import os import re from collections.abc import Iterable @@ -181,6 +182,31 @@ def offenders(rows: Iterable[StallRow]) -> tuple[str, ...]: ) +def _over_allowance(row: StallRow) -> int: + return row.count if row.allowed_by is None else max(0, row.count - row.allowed_by.max_count) + + +def junit_properties(spec: EventLoopStalls, rows: Iterable[StallRow]) -> tuple[tuple[str, float | str], ...]: + """The stall table as `perf_event_loop_*` properties for the dashboard: every hold over the threshold with the + allowlist's total as its limit, the holds past their allowance (limit 0, the case's one assertion), the longest + and the summed hold, and a `_breakdown` of holds per location in the shape the other metrics use.""" + table: Final = tuple(rows) + holds: Final = sum(row.count for row in table) + breakdown: Final = tuple( + {"label": f"{row.coroutine} {row.where}".strip(), "value": row.count, "share": row.count / holds} + for row in table + ) + return ( + ("perf_event_loop_holds", holds), + ("perf_event_loop_holds_limit", sum(known.max_count for known in spec.allowlist)), + ("perf_event_loop_offending_holds", sum(map(_over_allowance, table))), + ("perf_event_loop_offending_holds_limit", 0), + ("perf_event_loop_hold_seconds_max", max((row.max_seconds for row in table), default=0.0)), + ("perf_event_loop_hold_seconds_total", sum(row.total_seconds for row in table)), + *((("perf_event_loop_holds_breakdown", json.dumps(breakdown)),) if breakdown else ()), + ) + + def render_stalls(spec: EventLoopStalls, rows: Iterable[StallRow]) -> str: header: Final = ( f" event-loop holds >= {spec.max_hold_seconds:.2f} s in the gateway log (count total_s max_s where)" diff --git a/tests/e2e/perf/test_perf_cases.py b/tests/e2e/perf/test_perf_cases.py index 49b23ddfc57..3e5c3c522c8 100644 --- a/tests/e2e/perf/test_perf_cases.py +++ b/tests/e2e/perf/test_perf_cases.py @@ -29,6 +29,7 @@ from setup.event_loop import ( render_stalls, summarize, ) +from setup.event_loop import junit_properties as stall_properties from setup.observe import ( Observation, evaluate, @@ -158,7 +159,9 @@ def test_perf_case( item.user_properties[:] = [ entry for entry in item.user_properties if not entry[0].startswith("perf_") ] # rebind-ok: a --reruns retry replaces the first attempt's numbers instead of appending a second set - item.user_properties.extend(junit_properties(rows)) + item.user_properties.extend( + stall_properties(case.event_loop, stall_rows) if case.event_loop is not None else junit_properties(rows) + ) print(render(case, observation, rows)) if case.event_loop is not None: print(render_stalls(case.event_loop, stall_rows)) diff --git a/tests/e2e_harness/perf/setup/test_event_loop.py b/tests/e2e_harness/perf/setup/test_event_loop.py index 57868305716..e03e985709b 100644 --- a/tests/e2e_harness/perf/setup/test_event_loop.py +++ b/tests/e2e_harness/perf/setup/test_event_loop.py @@ -1,8 +1,10 @@ from __future__ import annotations +import math from typing import Final -from setup.event_loop import EventLoopStalls, KnownStall, offenders, parse_stalls, summarize +from pydantic import BaseModel, ConfigDict, TypeAdapter +from setup.event_loop import EventLoopStalls, KnownStall, junit_properties, offenders, parse_stalls, summarize _HANDLER: Final = ( "Executing None: (stall,) = parse_stalls(("INFO: plain log line", _HANDLER)) assert stall.coroutine == "RequestResponseCycle.run_asgi()" @@ -95,3 +105,29 @@ def test_a_collector_pass_explains_only_the_next_hold_and_only_one_at_least_as_l ("RequestResponseCycle.run_asgi()", False), ] assert summarize(_SPEC, parse_stalls((_GC_PASS,))) == () + + +def test_junit_properties_count_every_hold_and_only_the_holds_past_their_allowance_as_offending() -> None: + rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER, _HANDLER, _HANDLER))) + properties: Final = dict(junit_properties(_SPEC, rows)) + assert properties["perf_event_loop_holds"] == 6 + assert properties["perf_event_loop_holds_limit"] == 6 + assert properties["perf_event_loop_offending_holds"] == 3 + assert properties["perf_event_loop_offending_holds_limit"] == 0 + assert properties["perf_event_loop_hold_seconds_max"] == 0.731 + assert math.isclose(float(properties["perf_event_loop_hold_seconds_total"]), 0.302 + 2 * 0.412 + 3 * 0.731) + breakdown: Final = TypeAdapter(list[_Slice]).validate_json(str(properties["perf_event_loop_holds_breakdown"])) + assert [(slice.label, slice.value) for slice in breakdown] == [ + ("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 3), + ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2), + ("LifespanOn.main() uvicorn/lifespan/on.py:86", 1), + ] + assert math.isclose(sum(slice.share for slice in breakdown), 1.0) + + +def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None: + properties: Final = dict(junit_properties(_SPEC, ())) + assert properties["perf_event_loop_holds"] == 0 + assert properties["perf_event_loop_offending_holds"] == 0 + assert properties["perf_event_loop_hold_seconds_max"] == 0 + assert "perf_event_loop_holds_breakdown" not in properties From 41890bdf64145e17413430a0f21282a131a840c2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 18:24:22 -0700 Subject: [PATCH 037/235] fix(token_counter): stop re-downloading a HuggingFace tokenizer after a failed download (#47586) * fix(token_counter): stop re-downloading a HuggingFace tokenizer after a failed download Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(tokenizer): isolate HuggingFace tokenizer caches Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(token_counter): clear the traceback before re-raising a cached tokenizer load error Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(token_counter): retry failed HuggingFace tokenizer loads after the cache window Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(token_counter): start tokenizer retry TTL after a failure Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(utils): lock every HuggingFace tokenizer load and keep load failures as values Concurrent llama-2, llama-3 and cohere counts each started their own failed download because only the bundled tokenizers took a load lock. Every tokenizer kind now has one, so a burst during an outage makes a single attempt and two counts can no longer read the same expiring entry at once. A failed load is now remembered as a small value with the error message instead of the error object, so re-raising it no longer keeps the first failing request's frames alive. Adds tests for the concurrent outage and for fork-guard errors staying raised and unremembered. * fix(utils): keep Hugging Face tokenizer downloads unlocked and warn once per failed download Reverts the per-kind load lock so concurrent first downloads still run in parallel, as #47353 intends. A short lock around the failure cache read and write stops two counts racing on an expiring entry, and the tiktoken fallback warning now fires only when a download fails, not on every count that reuses the remembered failure. * fix(caching): read an entry another reader just expired as a cache miss Two threads reading the same expiring key in InMemoryCache could both pass the membership check, and the second then raised KeyError after the first removed it. get_cache now treats that key as a miss. This replaces the extra lock the tokenizer failure cache took, and the regression test forces the interleaving without timing. * fix(caching): treat a TTL another reader just removed as not expired InMemoryCache's expiry check read the TTL after a separate membership check, so a reader racing another reader's eviction raised KeyError. It now reads the TTL once and treats a missing one as not expired, and get_cache already reads the vanished entry as a miss. --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/caching/in_memory_cache.py | 10 +- litellm/constants.py | 3 + litellm/utils.py | 75 +++++++++-- tests/unit/caching/test_in_memory_cache.py | 41 +++++++ .../litellm_core_utils/test_token_counter.py | 116 +++++++++++++++++- .../unit/litellm_core_utils/test_tokenizer.py | 64 +++++++--- .../test_huggingface_chat_transformation.py | 5 + 7 files changed, 280 insertions(+), 34 deletions(-) diff --git a/litellm/caching/in_memory_cache.py b/litellm/caching/in_memory_cache.py index cb6426f64ff..1cc3e106aa0 100644 --- a/litellm/caching/in_memory_cache.py +++ b/litellm/caching/in_memory_cache.py @@ -95,7 +95,10 @@ class InMemoryCache(BaseCache): """ Check if a specific key is expired """ - return key in self.ttl_dict and self._clock() > self.ttl_dict[key] + try: + return self._clock() > self.ttl_dict[key] + except KeyError: + return False def _cache_keys(self) -> tuple[str, ...]: return tuple(self.cache_dict) # pyright: ignore[reportUnknownVariableType] # cache_dict is untyped legacy state @@ -227,7 +230,10 @@ class InMemoryCache(BaseCache): if key in self.cache_dict: if self.evict_element_if_expired(key): return None - original_cached_response: Final = self.cache_dict[key] + try: + original_cached_response: Final = self.cache_dict[key] + except KeyError: + return None try: cached_response = json.loads(original_cached_response) except Exception: diff --git a/litellm/constants.py b/litellm/constants.py index 1c1dfb9aa1c..6a503463a8c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -156,6 +156,9 @@ DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLIN DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1) DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250) HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0 +HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range( + "HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400 +) # Maximum wall-clock seconds a streaming response is allowed to run. # Streams exceeding this duration are terminated with a Timeout error. diff --git a/litellm/utils.py b/litellm/utils.py index 897138a940c..379eddb3155 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -71,6 +71,7 @@ from litellm.constants import ( DEFAULT_TRIM_RATIO, FUNCTION_DEFINITION_TOKEN_COUNT, HF_CONFIG_FETCH_TIMEOUT_SECONDS, + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, INITIAL_RETRY_DELAY, JITTER, MAX_RETRY_DELAY, @@ -2597,19 +2598,14 @@ def _select_tokenizer_helper(model: str) -> SelectTokenizerResponse: try: result: Final = _return_huggingface_tokenizer(model) - if result is not None: + if result is not None and not isinstance(result, _HuggingFaceTokenizerLoadFailure): return result except Exception as e: from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking if isinstance(e, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): raise - verbose_logger.warning( - "Falling back to tiktoken for %s; token counts may be approximate. " - "For Python Hugging Face tokenization, install tokenizers and huggingface-hub. Error: %s", - json.dumps(redact_secrets(model)), - json.dumps(redact_secrets(str(e))), - ) + _warn_huggingface_tokenizer_fallback(model, str(e)) # default - tiktoken return _return_openai_tokenizer(model) @@ -2649,15 +2645,41 @@ def huggingface_tokenizer_kind(model: str) -> HuggingFaceTokenizerKind | None: return None -def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | None: +@dataclass(frozen=True, slots=True) +class _HuggingFaceTokenizerLoadFailure: + reason: str + + +def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: + verbose_logger.warning( + "Falling back to tiktoken for %s; token counts may be approximate. " + "For Python Hugging Face tokenization, install tokenizers and huggingface-hub. Error: %s", + json.dumps(redact_secrets(model)), + json.dumps(redact_secrets(reason)), + ) + + +def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: kind: Final = huggingface_tokenizer_kind(model) if kind is None: return None + backend: Final = _huggingface_tokenizer_backend() + key: Final = _huggingface_tokenizer_failure_key(kind, backend) with _BUNDLED_TOKENIZER_LOAD_LOCKS.get(kind, nullcontext()): - tokenizer: Final = _load_huggingface_tokenizer(kind, _huggingface_tokenizer_backend()) + cached_failure: Final = _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.get_cache(key) + if isinstance(cached_failure, _HuggingFaceTokenizerLoadFailure): + return cached_failure + tokenizer: Final = _load_huggingface_tokenizer_recording_failure(kind, backend, key) + if isinstance(tokenizer, _HuggingFaceTokenizerLoadFailure): + _warn_huggingface_tokenizer_fallback(model, tokenizer.reason) + return tokenizer return {"type": "huggingface_tokenizer", "tokenizer": tokenizer} +def _huggingface_tokenizer_failure_key(kind: HuggingFaceTokenizerKind, backend: Decision) -> str: + return f"{kind}:{backend.name}" + + @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE) def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decision) -> HuggingFace: """One tokenizer per kind and backend; `backend` is the cache key, the dispatch re-derives it.""" @@ -2680,6 +2702,21 @@ def _load_huggingface_tokenizer(kind: HuggingFaceTokenizerKind, backend: Decisio assert_never(kind) +def _load_huggingface_tokenizer_recording_failure( + kind: HuggingFaceTokenizerKind, backend: Decision, key: str +) -> HuggingFace | _HuggingFaceTokenizerLoadFailure: + try: + return _load_huggingface_tokenizer(kind, backend) + except Exception as error: + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted, ProcessReservedForForking + + if isinstance(error, (ForkedAfterNativeRuntimeStarted, ProcessReservedForForking)): + raise + failure: Final = _HuggingFaceTokenizerLoadFailure(reason=str(error)) + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.set_cache(key, failure, ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS) + return failure + + def encode(model="", text="", custom_tokenizer: dict | None = None): """ Encodes the given text using the specified model. @@ -8295,6 +8332,26 @@ def trim_messages( from litellm.caching.in_memory_cache import InMemoryCache +class _HuggingFaceTokenizerLoadFailureCache(Protocol): + def get_cache(self, key: str) -> object | None: ... + + def set_cache(self, key: str, value: object, ttl: float | None = None) -> None: ... + + def flush_cache(self) -> None: ... + + +_HUGGINGFACE_TOKENIZER_LOAD_FAILURES: Final[_HuggingFaceTokenizerLoadFailureCache] = ( + cast( # cast-ok: cache methods are untyped + _HuggingFaceTokenizerLoadFailureCache, + InMemoryCache( + max_size_in_memory=16, + default_ttl=HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, + clock=lambda: time.monotonic(), + ), + ) +) + + class AvailableModelsCache(InMemoryCache): def __init__(self, ttl_seconds: int = 300, max_size: int = 1000): super().__init__(ttl_seconds, max_size) diff --git a/tests/unit/caching/test_in_memory_cache.py b/tests/unit/caching/test_in_memory_cache.py index 214f4ecf301..75232476ecd 100644 --- a/tests/unit/caching/test_in_memory_cache.py +++ b/tests/unit/caching/test_in_memory_cache.py @@ -49,6 +49,47 @@ async def test_async_increment_delegates_to_locked_sync_path(): assert cache.get_cache("counter") == 5 +class _AnotherReaderExpiresTheEntryFirstCache(InMemoryCache): + other_reader_started = False + other_reader_result: object = "not run" + + def evict_element_if_expired(self, key: str) -> bool: + if not self.other_reader_started: + self.other_reader_started = True + self.other_reader_result = self.get_cache(key) + return super().evict_element_if_expired(key) + + +def test_get_cache_reads_an_entry_another_reader_just_expired_as_a_miss() -> None: + cache: Final = _AnotherReaderExpiresTheEntryFirstCache() + cache.set_cache("tokenizer_failure", "HF unavailable", ttl=-1) + + assert cache.get_cache("tokenizer_failure") is None + assert cache.other_reader_result is None + + +class _ClockThatLetsAnotherReaderRunFirst: + cache: InMemoryCache | None = None + other_reader_result: object = "not run" + + def __call__(self) -> float: + cache: Final = self.cache + if cache is not None: + self.cache = None + self.other_reader_result = cache.get_cache("tokenizer_failure") + return 100.0 + + +def test_get_cache_reads_an_entry_another_reader_expires_during_the_expiry_check_as_a_miss() -> None: + clock: Final = _ClockThatLetsAnotherReaderRunFirst() + cache: Final = InMemoryCache(clock=clock) + cache.set_cache("tokenizer_failure", "HF unavailable", ttl=-1) + clock.cache = cache + + assert cache.get_cache("tokenizer_failure") is None + assert clock.other_reader_result is None + + def test_in_memory_openai_obj_cache(): from openai import OpenAI diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 309cbd33131..194d7995735 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -14,7 +14,7 @@ from concurrent.futures import Future, ThreadPoolExecutor, wait from itertools import accumulate, chain from pathlib import Path from typing import Final -from unittest.mock import MagicMock +from unittest.mock import MagicMock, sentinel import anyio.to_thread import pytest @@ -27,7 +27,7 @@ import litellm from litellm import decode, encode, get_modified_max_tokens from litellm import token_counter as token_counter_old import litellm.constants -from litellm.constants import TOKEN_COUNTER_MAX_CONCURRENT_COUNTS +from litellm.constants import HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS, TOKEN_COUNTER_MAX_CONCURRENT_COUNTS from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.token_counter import ( _encoding_count, @@ -940,10 +940,17 @@ def test_token_counter(): import unittest -from litellm.utils import _load_huggingface_tokenizer, _select_tokenizer_helper, claude_json_str, encoding +from litellm.utils import ( + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES, + _load_huggingface_tokenizer, + _select_tokenizer_helper, + claude_json_str, + encoding, +) # Clear the cache at module load to ensure clean state _load_huggingface_tokenizer.cache_clear() +_HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() class TestTokenizerSelection(unittest.TestCase): @@ -955,6 +962,7 @@ class TestTokenizerSelection(unittest.TestCase): --dist=loadscope (tests from same file run on same worker). """ _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() @patch("litellm.utils.tokenizer_dispatch.from_pretrained") def test_llama3_tokenizer_api_failure(self, mock_from_pretrained): @@ -1038,6 +1046,69 @@ class TestTokenizerSelection(unittest.TestCase): finally: monkeypatch.undo() + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") + def test_failed_huggingface_tokenizer_download_is_not_retried_on_every_count( + self, mock_from_pretrained: MagicMock + ): + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + try: + with patch("litellm.utils.time.monotonic", return_value=0.0): + mock_from_pretrained.side_effect = RuntimeError("HF unavailable") + model: Final = "databricks/databricks-meta-llama-3-3-70b-instruct" + text: Final = "Count the tokens for this message." + expected: Final = len(encoding.encode(text, disallowed_special=())) + + first_count: Final = litellm.token_counter(model=model, text=text) + first_attempts: Final = mock_from_pretrained.call_count + second_count: Final = litellm.token_counter(model=model, text=text) + + assert first_attempts > 0 + assert mock_from_pretrained.call_count == first_attempts + assert first_count == expected + assert second_count == expected + finally: + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + + @patch("litellm.utils.time.monotonic", return_value=299.0) + @patch("litellm.utils.tokenizer_dispatch.from_pretrained") + def test_failed_huggingface_tokenizer_download_is_retried_after_retry_window( + self, mock_from_pretrained: MagicMock, mock_monotonic: MagicMock + ): + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + try: + def from_pretrained_side_effect(_: str) -> object: + if mock_from_pretrained.call_count == 1: + mock_monotonic.return_value = 322.0 + raise RuntimeError("HF unavailable") + return sentinel.tokenizer + + mock_from_pretrained.side_effect = from_pretrained_side_effect + model: Final = "databricks/databricks-meta-llama-3-3-70b-instruct" + + first_result: Final = _select_tokenizer_helper(model) + assert first_result["type"] == "openai_tokenizer" + mock_monotonic.return_value = 323.0 + second_result: Final = _select_tokenizer_helper(model) + assert second_result["type"] == "openai_tokenizer", "tokenizer was retried at t=323" + assert mock_from_pretrained.call_count == 1 + + mock_monotonic.return_value = 322.0 + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS - 1 + third_result: Final = _select_tokenizer_helper(model) + assert third_result["type"] == "openai_tokenizer" + assert mock_from_pretrained.call_count == 1 + + mock_monotonic.return_value = 322.0 + HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS + 1 + retried_result: Final = _select_tokenizer_helper(model) + assert retried_result["type"] == "huggingface_tokenizer" + assert retried_result["tokenizer"] is sentinel.tokenizer + assert mock_from_pretrained.call_count == 2 + finally: + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + _CODE_SAMPLE: Final = " for item in items:\n total += item.price * item.quantity\n" * 20 @@ -1104,6 +1175,7 @@ def test_disable_hf_tokenizer_download_keeps_deepseek_v4_on_cl100k(monkeypatch: def test_concurrent_first_deepseek_counts_share_one_loaded_tokenizer(): _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() start: Final = threading.Barrier(8) def first_tokenizer(_: int) -> object: @@ -1116,6 +1188,44 @@ def test_concurrent_first_deepseek_counts_share_one_loaded_tokenizer(): assert len({id(tokenizer) for tokenizer in tokenizers}) == 1 +def test_remembered_tokenizer_failure_warns_once_instead_of_on_every_count(caplog: pytest.LogCaptureFixture): + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + model: Final = "databricks/databricks-meta-llama-3-3-70b-instruct" + try: + with ( + patch("litellm.utils.tokenizer_dispatch.from_pretrained", side_effect=RuntimeError("HF unavailable")), + caplog.at_level("WARNING", logger="LiteLLM"), + ): + types: Final = tuple(_select_tokenizer_helper(model)["type"] for _ in range(3)) + fallback_warnings: Final = [r for r in caplog.records if "token counts may be approximate" in r.getMessage()] + assert types == ("openai_tokenizer",) * 3 + assert len(fallback_warnings) == 1 + finally: + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + + +def test_fork_guard_error_while_loading_a_tokenizer_is_raised_and_not_remembered(): + from litellm.rust_bridge.fork_guard import ForkedAfterNativeRuntimeStarted + + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + model: Final = "databricks/databricks-meta-llama-3-3-70b-instruct" + forked: Final = ForkedAfterNativeRuntimeStarted("native runtime started before fork") + try: + with patch("litellm.utils.tokenizer_dispatch.from_pretrained", side_effect=forked): + with pytest.raises(ForkedAfterNativeRuntimeStarted): + _select_tokenizer_helper(model) + with patch("litellm.utils.tokenizer_dispatch.from_pretrained", return_value=sentinel.tokenizer): + loaded: Final = _select_tokenizer_helper(model) + assert loaded["type"] == "huggingface_tokenizer" + assert loaded["tokenizer"] is sentinel.tokenizer + finally: + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + + def test_token_counter_with_anthropic_tool_use(): """ Test that _count_anthropic_content() correctly handles tool_use blocks. diff --git a/tests/unit/litellm_core_utils/test_tokenizer.py b/tests/unit/litellm_core_utils/test_tokenizer.py index 354c91920fc..61e797d7166 100644 --- a/tests/unit/litellm_core_utils/test_tokenizer.py +++ b/tests/unit/litellm_core_utils/test_tokenizer.py @@ -406,16 +406,26 @@ def test_huggingface_encoding_exposes_the_tokenizers_lookup_and_mutation_surface @pytest.mark.parametrize("log_level", ["WARNING", "ERROR"]) def test_missing_python_tokenizer_warns_before_approximate_count(caplog, monkeypatch, log_level): from unittest.mock import patch - from litellm.utils import _load_huggingface_tokenizer, _select_tokenizer_helper + + from litellm.utils import ( + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES, + _load_huggingface_tokenizer, + _select_tokenizer_helper, + ) monkeypatch.setenv("LITELLM_RUST", "false") _load_huggingface_tokenizer.cache_clear() - with caplog.at_level(log_level, logger="LiteLLM"), patch.dict(sys.modules, {"tokenizers": None}): - result = _select_tokenizer_helper("llama-2") - assert result["type"] == "openai_tokenizer" - assert result["tokenizer"].encode("hello") - assert ("token counts may be approximate" in caplog.text) is (log_level == "WARNING") - assert ("install tokenizers" in caplog.text) is (log_level == "WARNING") + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + try: + with caplog.at_level(log_level, logger="LiteLLM"), patch.dict(sys.modules, {"tokenizers": None}): + result = _select_tokenizer_helper("llama-2") + assert result["type"] == "openai_tokenizer" + assert result["tokenizer"].encode("hello") + assert ("token counts may be approximate" in caplog.text) is (log_level == "WARNING") + assert ("install tokenizers" in caplog.text) is (log_level == "WARNING") + finally: + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() @pytest.mark.parametrize("python_installed", [False, True]) @@ -471,10 +481,16 @@ def test_added_token_return_annotation_resolves_without_optional_import(python_i def test_tokenizer_fallback_logs_safe_diagnostic_context(caplog, monkeypatch): from types import SimpleNamespace from unittest.mock import patch - from litellm.utils import _load_huggingface_tokenizer, _select_tokenizer_helper + + from litellm.utils import ( + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES, + _load_huggingface_tokenizer, + _select_tokenizer_helper, + ) monkeypatch.setenv("LITELLM_RUST", "false") _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() model = "llama-2\r\nforged-model\x1b[31m\u2028\u2029" secret = "sk-" + "x" * 48 failure = OSError("download failed\r\nforged-error\x1b[31m api_key=" + secret) @@ -483,18 +499,26 @@ def test_tokenizer_fallback_logs_safe_diagnostic_context(caplog, monkeypatch): raise failure dependency = SimpleNamespace(Tokenizer=SimpleNamespace(from_pretrained=fail_download)) - with caplog.at_level("WARNING", logger="LiteLLM"), patch.dict(sys.modules, {"tokenizers": dependency}): - result = _select_tokenizer_helper(model) - assert result["type"] == "openai_tokenizer" - assert result["tokenizer"].encode("hello") - message = next(record.getMessage() for record in caplog.records if "token counts may be approximate" in record.getMessage()) - assert "llama-2" in message - assert "download failed" in message - assert "forged-model" in message and "forged-error" in message - assert message.isascii() and message.isprintable() - assert secret not in message - assert "REDACTED" in message - assert "install tokenizers and huggingface-hub" in message + try: + with caplog.at_level("WARNING", logger="LiteLLM"), patch.dict(sys.modules, {"tokenizers": dependency}): + result = _select_tokenizer_helper(model) + assert result["type"] == "openai_tokenizer" + assert result["tokenizer"].encode("hello") + message = next( + record.getMessage() + for record in caplog.records + if "token counts may be approximate" in record.getMessage() + ) + assert "llama-2" in message + assert "download failed" in message + assert "forged-model" in message and "forged-error" in message + assert message.isascii() and message.isprintable() + assert secret not in message + assert "REDACTED" in message + assert "install tokenizers and huggingface-hub" in message + finally: + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() @pytest.mark.parametrize("python_installed", [False, True]) diff --git a/tests/unit/llms/huggingface/chat/test_huggingface_chat_transformation.py b/tests/unit/llms/huggingface/chat/test_huggingface_chat_transformation.py index 2f1b3764610..04c2ce153c6 100644 --- a/tests/unit/llms/huggingface/chat/test_huggingface_chat_transformation.py +++ b/tests/unit/llms/huggingface/chat/test_huggingface_chat_transformation.py @@ -10,6 +10,7 @@ import respx import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.llms.huggingface.common_utils import _fetch_inference_provider_mapping +from litellm.utils import _HUGGINGFACE_TOKENIZER_LOAD_FAILURES, _load_huggingface_tokenizer HUGGINGFACE_MODEL: Final = "meta-llama/Meta-Llama-3-8B-Instruct" HUGGINGFACE_ROUTER_MODEL: Final = "meta-llama/Meta-Llama-3-8B-Instruct-Turbo" @@ -83,9 +84,13 @@ def isolate_huggingface_environment(monkeypatch: pytest.MonkeyPatch) -> Iterator monkeypatch.delenv("HF_API_BASE", raising=False) monkeypatch.delenv("HUGGINGFACE_API_BASE", raising=False) monkeypatch.delenv("HUGGINGFACE_API_KEY", raising=False) + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() _fetch_inference_provider_mapping.cache_clear() yield _fetch_inference_provider_mapping.cache_clear() + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() PROVIDER_MAPPING_RESPONSE = { From c515f80f8260580f6fd5e07688535f77e4cdf6f2 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:46:03 +0000 Subject: [PATCH 038/235] ci(cost map): delete the model prices JSON schema file (#47079) * chore(cost map): delete the unused model prices JSON schema Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(cost map): remove interim schema sync compatibility Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/scripts/classify_changes.sh | 2 +- ci_cd/cost_map_guard.py | 6 +- model_prices_and_context_window.schema.json | 1312 ------------------- tests/unit/test_circleci_path_filter.py | 1 - tests/unit/test_cost_map_guard.py | 7 - 5 files changed, 3 insertions(+), 1325 deletions(-) delete mode 100644 model_prices_and_context_window.schema.json diff --git a/.circleci/scripts/classify_changes.sh b/.circleci/scripts/classify_changes.sh index d2ab3478397..a33695d0dbe 100755 --- a/.circleci/scripts/classify_changes.sh +++ b/.circleci/scripts/classify_changes.sh @@ -33,7 +33,7 @@ while IFS= read -r file || [ -n "$file" ]; do *) has_backend=true ;; esac case "$file" in - model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json) + model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json) has_cost_map=true ;; tests/unit/proxy/*) : ;; *) outside_cost_map_set=true ;; diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py index 932da84e5d7..c1aeaabc5d2 100644 --- a/ci_cd/cost_map_guard.py +++ b/ci_cd/cost_map_guard.py @@ -4,8 +4,7 @@ Every pull request whose diff against its merge base touches one of the two cost checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always -get the file checks and additionally may only touch those two files or the legacy schema -path and may only add or update models. +get the file checks and additionally may only touch those two files and may only add or update models. """ from __future__ import annotations @@ -21,7 +20,6 @@ from typing import Final COST_MAP_PATH: Final = "model_prices_and_context_window.json" BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json" -LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json" GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH) BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_" SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"}) @@ -110,7 +108,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str *( f"bot PRs may only change the cost map files, not {path}" for path in changed_files - if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH + if path not in GUARDED_PATHS ), *(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map), *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields), diff --git a/model_prices_and_context_window.schema.json b/model_prices_and_context_window.schema.json deleted file mode 100644 index d4dfeb25429..00000000000 --- a/model_prices_and_context_window.schema.json +++ /dev/null @@ -1,1312 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "LiteLLM model_prices_and_context_window.json", - "description": "Schema for LiteLLM's model price and context window registry (https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json). Every top-level key except 'sample_spec' and 'fallback_generalizations' is a model id, optionally prefixed with its provider (e.g. 'azure/gpt-5.4'), mapping to a model entry. All costs are USD per unit. New optional fields are added regularly, so consumers should ignore unknown fields rather than reject them.", - "type": "object", - "properties": { - "sample_spec": { - "type": "object", - "description": "Documentation placeholder illustrating the entry shape; not a real model and not schema-conformant (several values are prose)." - }, - "fallback_generalizations": { - "type": "object", - "description": "Regex rules that generalize unknown model ids to known families; not a model entry.", - "properties": { - "rules": { - "type": "array", - "items": { - "type": "object", - "properties": { - "name": { - "type": "string" - }, - "pattern": { - "type": "string" - }, - "description": { - "type": "string" - } - }, - "required": [ - "name", - "pattern" - ], - "additionalProperties": true - } - } - }, - "additionalProperties": false - } - }, - "additionalProperties": { - "$ref": "#/$defs/modelEntry" - }, - "$defs": { - "modelEntry": { - "type": "object", - "description": "Pricing, limits, and capability flags for one model. Fields other than litellm_provider are optional; boolean capability flags are simply omitted when unknown or false.", - "required": [ - "litellm_provider" - ], - "properties": { - "annotation_cost_per_page": { - "type": "number", - "minimum": 0 - }, - "annotation_cost_per_page_batches": { - "type": "number", - "minimum": 0 - }, - "audio_transcription_config": { - "type": "string" - }, - "bedrock_converse_supports_strict_tools": { - "type": "boolean" - }, - "bedrock_output_config_effort_ceiling": { - "type": "string", - "description": "Highest reasoning effort the Bedrock output_config accepts for this model.", - "enum": [ - "low", - "medium", - "high", - "max", - "xhigh" - ] - }, - "cache_creation_input_audio_token_cost": { - "type": "number", - "minimum": 0 - }, - "cache_creation_input_token_cost": { - "type": "number", - "minimum": 0, - "description": "USD per token written to the provider's prompt cache." - }, - "cache_creation_input_token_cost_above_100k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_100k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_1hr": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_1hr_above_100k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_1hr_above_200k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_200k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_272k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_272k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_272k_tokens_decisions": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_above_272k_tokens_flex": { - "type": "number", - "minimum": 0, - "description": "Flex service-tier rate for the same-named base field." - }, - "cache_creation_input_token_cost_above_272k_tokens_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "cache_creation_input_token_cost_above_272k_tokens_ultrafast": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_creation_input_token_cost_batches": { - "type": "number", - "minimum": 0 - }, - "cache_creation_input_token_cost_decisions": { - "type": "number", - "minimum": 0 - }, - "cache_creation_input_token_cost_flex": { - "type": "number", - "minimum": 0, - "description": "Flex service-tier rate for the same-named base field." - }, - "cache_creation_input_token_cost_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "cache_creation_input_token_cost_ultrafast": { - "type": "number", - "minimum": 0 - }, - "cache_read_input_audio_token_cost": { - "type": "number", - "minimum": 0 - }, - "cache_read_input_image_token_cost": { - "type": "number", - "minimum": 0 - }, - "cache_read_input_token_cost": { - "type": "number", - "minimum": 0, - "description": "USD per prompt token served from the provider's prompt cache." - }, - "cache_read_input_token_cost_above_100k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_above_100k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_above_200k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_above_200k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_above_200k_tokens_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "cache_read_input_token_cost_above_272k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_above_272k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_above_272k_tokens_decisions": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_above_272k_tokens_flex": { - "type": "number", - "minimum": 0, - "description": "Flex service-tier rate for the same-named base field." - }, - "cache_read_input_token_cost_above_272k_tokens_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "cache_read_input_token_cost_above_272k_tokens_ultrafast": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_above_512k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "cache_read_input_token_cost_balanced": { - "type": "number", - "minimum": 0 - }, - "cache_read_input_token_cost_batches": { - "type": "number", - "minimum": 0 - }, - "cache_read_input_token_cost_decisions": { - "type": "number", - "minimum": 0 - }, - "cache_read_input_token_cost_flex": { - "type": "number", - "minimum": 0, - "description": "Flex service-tier rate for the same-named base field." - }, - "cache_read_input_token_cost_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "cache_read_input_token_cost_ultrafast": { - "type": "number", - "minimum": 0 - }, - "citation_cost_per_token": { - "type": "number", - "minimum": 0 - }, - "code_interpreter_cost_per_session": { - "type": "number", - "minimum": 0 - }, - "comment": { - "type": "string" - }, - "cost_per_second": { - "type": "number", - "minimum": 0 - }, - "default_reasoning_effort": { - "type": "string", - "description": "Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.", - "enum": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh" - ] - }, - "deprecation_date": { - "type": "string", - "description": "Date the provider deprecates the model, YYYY-MM-DD.", - "format": "date", - "pattern": "^\\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\\d|3[01])$" - }, - "explicit_cache_read_input_token_cost": { - "type": "number", - "minimum": 0 - }, - "gemini_audio_only_live": { - "type": "boolean" - }, - "gemini_native_audio": { - "type": "boolean" - }, - "google_maps_grounding_cost_per_query": { - "type": "number", - "minimum": 0, - "description": "USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit." - }, - "guardrail_cost_per_unit": { - "type": "object", - "description": "USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).", - "additionalProperties": { - "type": "number", - "minimum": 0 - } - }, - "input_cost_per_audio_per_second": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_audio_per_second_above_128k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_audio_token": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_audio_token_batches": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_audio_token_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "input_cost_per_character": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_character_above_128k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_image": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_image_above_128k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_image_token": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_image_token_batches": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_pixel": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_query": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_request": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_second": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_text_message": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_token": { - "type": "number", - "minimum": 0, - "description": "USD per prompt token." - }, - "input_cost_per_token_above_100k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_100k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_128k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_200k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_200k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_200k_tokens_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "input_cost_per_token_above_272k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_272k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_272k_tokens_decisions": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_272k_tokens_flex": { - "type": "number", - "minimum": 0, - "description": "Flex service-tier rate for the same-named base field." - }, - "input_cost_per_token_above_272k_tokens_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "input_cost_per_token_above_272k_tokens_ultrafast": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_above_512k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_token_balanced": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_token_batches": { - "type": "number", - "minimum": 0, - "description": "USD per prompt token via the provider's batch API." - }, - "input_cost_per_token_cache_hit": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_token_decisions": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_token_flex": { - "type": "number", - "minimum": 0, - "description": "Flex service-tier rate for the same-named base field." - }, - "input_cost_per_token_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "input_cost_per_token_ultrafast": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_video_per_second": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_video_per_second_above_128k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_video_per_second_above_15s_interval": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_video_per_second_above_8s_interval": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "input_cost_per_video_token": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_video_token_batches": { - "type": "number", - "minimum": 0 - }, - "input_dbu_cost_per_token": { - "type": "number", - "minimum": 0 - }, - "litellm_provider": { - "type": "string", - "description": "LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers." - }, - "max_input_tokens": { - "type": "integer", - "minimum": 0, - "description": "Maximum prompt/context tokens the model accepts." - }, - "max_output_tokens": { - "type": "integer", - "minimum": 0, - "description": "Maximum tokens the model can generate in one response." - }, - "max_tokens": { - "type": "integer", - "minimum": 0, - "description": "Legacy field: max output tokens if the provider specifies it, else max input tokens." - }, - "metadata": { - "type": "object", - "description": "Free-form notes about the entry (e.g. pricing derivation)." - }, - "mode": { - "type": "string", - "description": "Primary API surface / task type of the model.", - "enum": [ - "audio_speech", - "audio_transcription", - "chat", - "completion", - "embedding", - "evaluation", - "extraction", - "guardrail", - "image_edit", - "image_generation", - "moderation", - "ocr", - "realtime", - "rerank", - "responses", - "search", - "vector_store", - "video_generation" - ] - }, - "ocr_cost_per_credit": { - "type": "number", - "minimum": 0 - }, - "ocr_cost_per_page": { - "type": "number", - "minimum": 0 - }, - "ocr_cost_per_page_batches": { - "type": "number", - "minimum": 0 - }, - "off_peak_pricing": { - "type": "object", - "description": "Rates that replace the same-named base fields while the request falls inside the stated UTC windows.", - "properties": { - "hours_utc": { - "description": "UTC \"HH:MM-HH:MM\" window, or a list of them; a window may wrap past midnight.", - "oneOf": [ - { - "type": "string", - "pattern": "^([01]\\d|2[0-3]):[0-5]\\d-([01]\\d|2[0-3]):[0-5]\\d$" - }, - { - "type": "array", - "items": { - "type": "string", - "pattern": "^([01]\\d|2[0-3]):[0-5]\\d-([01]\\d|2[0-3]):[0-5]\\d$" - }, - "minItems": 1 - } - ] - }, - "windows": { - "type": "array", - "items": { - "type": "object", - "properties": { - "hours_utc": { - "description": "UTC \"HH:MM-HH:MM\" window, or a list of them; a window may wrap past midnight.", - "oneOf": [ - { - "type": "string", - "pattern": "^([01]\\d|2[0-3]):[0-5]\\d-([01]\\d|2[0-3]):[0-5]\\d$" - }, - { - "type": "array", - "items": { - "type": "string", - "pattern": "^([01]\\d|2[0-3]):[0-5]\\d-([01]\\d|2[0-3]):[0-5]\\d$" - }, - "minItems": 1 - } - ] - }, - "weekdays": { - "type": "array", - "description": "ISO-8601 weekday numbers (1 = Monday .. 7 = Sunday) or English day names the window applies on.", - "items": { - "oneOf": [ - { - "type": "integer", - "minimum": 1, - "maximum": 7 - }, - { - "type": "string", - "pattern": "(?i)^(mon|monday|tue|tues|tuesday|wed|wednesday|thu|thur|thurs|thursday|fri|friday|sat|saturday|sun|sunday)$" - } - ] - }, - "minItems": 1 - } - }, - "required": [ - "hours_utc" - ], - "additionalProperties": false - }, - "minItems": 1 - }, - "weekday_timezone": { - "type": "string", - "description": "IANA zone the weekdays of each window are read on; defaults to UTC." - }, - "input_cost_per_token": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_token": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_reasoning_token": { - "type": "number", - "minimum": 0 - }, - "cache_read_input_token_cost": { - "type": "number", - "minimum": 0 - }, - "cache_creation_input_token_cost": { - "type": "number", - "minimum": 0 - } - }, - "anyOf": [ - { - "required": [ - "hours_utc" - ] - }, - { - "required": [ - "windows" - ] - } - ], - "additionalProperties": false - }, - "output_cost_per_audio_token": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_character": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_character_above_128k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_image": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_0.5K": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_1024": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_1536": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_1K": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_2K": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_4K": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_512": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_token": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_image_token_batches": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_pixel": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_reasoning_token": { - "type": "number", - "minimum": 0, - "description": "USD per reasoning/thinking token, when billed separately." - }, - "output_cost_per_second": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_second_1080p": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_second_2k": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_second_480p": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_second_4k": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_second_720p": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_second_768p": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_token": { - "type": "number", - "minimum": 0, - "description": "USD per generated token." - }, - "output_cost_per_token_above_100k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_100k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_128k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_200k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_200k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_200k_tokens_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "output_cost_per_token_above_272k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_272k_tokens_batches": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_272k_tokens_decisions": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_272k_tokens_flex": { - "type": "number", - "minimum": 0, - "description": "Flex service-tier rate for the same-named base field." - }, - "output_cost_per_token_above_272k_tokens_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "output_cost_per_token_above_272k_tokens_ultrafast": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_above_512k_tokens": { - "type": "number", - "minimum": 0, - "description": "Rate applied once the prompt exceeds the token threshold in the field name." - }, - "output_cost_per_token_balanced": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_token_batches": { - "type": "number", - "minimum": 0, - "description": "USD per generated token via the provider's batch API." - }, - "output_cost_per_token_decisions": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_token_flex": { - "type": "number", - "minimum": 0, - "description": "Flex service-tier rate for the same-named base field." - }, - "output_cost_per_token_priority": { - "type": "number", - "minimum": 0, - "description": "Priority service-tier rate for the same-named base field." - }, - "output_cost_per_token_ultrafast": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_video_per_second": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_video_token": { - "type": "number", - "minimum": 0 - }, - "output_dbu_cost_per_token": { - "type": "number", - "minimum": 0 - }, - "output_vector_size": { - "type": "integer", - "minimum": 0, - "description": "Embedding dimension for embedding models." - }, - "prompt_cache_min_tokens": { - "type": "integer", - "minimum": 0, - "description": "Smallest prefix the provider will actually cache; absent means the provider default applies." - }, - "provider_specific_entry": { - "type": "object", - "description": "Provider-internal routing hints (e.g. bedrock_invocation_schema)." - }, - "reasoning_effort_levels": { - "type": "array", - "description": "Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.", - "items": { - "type": "string", - "enum": [ - "none", - "minimal", - "low", - "medium", - "high", - "xhigh", - "max" - ] - } - }, - "regional_endpoint_uplift_multiplier": { - "type": "number", - "minimum": 1, - "description": "Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%)." - }, - "regional_processing_uplift_multiplier_eu": { - "type": "number", - "minimum": 1, - "description": "Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%)." - }, - "regional_processing_uplift_multiplier_us": { - "type": "number", - "minimum": 1, - "description": "Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%)." - }, - "rpm": { - "type": "integer", - "minimum": 0, - "description": "Provider default requests-per-minute limit." - }, - "search_context_cost_per_query": { - "type": "object", - "description": "USD cost per web search query, keyed by search context size.", - "properties": { - "search_context_size_low": { - "type": "number", - "minimum": 0 - }, - "search_context_size_medium": { - "type": "number", - "minimum": 0 - }, - "search_context_size_high": { - "type": "number", - "minimum": 0 - } - }, - "additionalProperties": false - }, - "search_free_requests": { - "type": "object", - "description": "Free Google Search grounding requests per period, shared across models in the same shared_group.", - "required": [ - "count", - "period", - "shared_group" - ], - "additionalProperties": false, - "properties": { - "count": { - "type": "integer", - "minimum": 0 - }, - "period": { - "type": "string", - "enum": [ - "day", - "month" - ] - }, - "shared_group": { - "type": "string", - "pattern": "\\S" - } - } - }, - "source": { - "type": "string", - "description": "URL of the provider pricing/model page this entry was taken from." - }, - "supported_audio_formats": { - "type": "array", - "description": "Audio container formats the model can return.", - "items": { - "type": "string", - "enum": [ - "mp3", - "wav" - ] - } - }, - "supported_endpoints": { - "type": "array", - "description": "OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.", - "items": { - "type": "string" - } - }, - "supported_modalities": { - "type": "array", - "description": "Input modalities the model accepts.", - "items": { - "type": "string", - "enum": [ - "text", - "image", - "audio", - "video" - ] - } - }, - "supported_output_modalities": { - "type": "array", - "description": "Output modalities the model can produce.", - "items": { - "type": "string", - "enum": [ - "text", - "image", - "audio", - "video", - "code" - ] - } - }, - "supported_regions": { - "type": "array", - "description": "Cloud regions the model is available in ('global' or region ids).", - "items": { - "type": "string" - } - }, - "supports_adaptive_thinking": { - "type": "boolean" - }, - "supports_anthropic_compaction": { - "type": "boolean" - }, - "supports_anthropic_thinking_payload": { - "type": "boolean" - }, - "supports_assistant_prefill": { - "type": "boolean" - }, - "supports_audio_input": { - "type": "boolean" - }, - "supports_audio_output": { - "type": "boolean" - }, - "supports_bedrock_converse_tool_result_images": { - "type": "boolean" - }, - "supports_bedrock_native_structured_output": { - "type": "boolean" - }, - "supports_bedrock_runtime_chat_completions_inline_reasoning": { - "type": "boolean" - }, - "supports_bedrock_runtime_chat_completions_response_format": { - "type": "boolean" - }, - "supports_bedrock_runtime_chat_completions_tools_with_reasoning": { - "type": "boolean" - }, - "supports_box_questions": { - "type": "boolean" - }, - "supports_computer_use": { - "type": "boolean" - }, - "supports_embedding_image_input": { - "type": "boolean" - }, - "supports_fast_mode": { - "type": "boolean" - }, - "supports_forced_tool_use": { - "type": "boolean" - }, - "supports_function_calling": { - "type": "boolean" - }, - "supports_image_input": { - "type": "boolean" - }, - "supports_image_size": { - "type": "boolean" - }, - "supports_legacy_thinking": { - "type": "boolean" - }, - "supports_low_reasoning_effort": { - "type": "boolean" - }, - "supports_max_reasoning_effort": { - "type": "boolean" - }, - "supports_mid_conversation_system": { - "type": "boolean" - }, - "supports_minimal_reasoning_effort": { - "type": "boolean" - }, - "supports_multimodal": { - "type": "boolean" - }, - "supports_native_streaming": { - "type": "boolean" - }, - "supports_native_structured_output": { - "type": "boolean" - }, - "supports_none_reasoning_effort": { - "type": "boolean" - }, - "supports_nova_canvas_image_edit": { - "type": "boolean" - }, - "supports_output_config": { - "type": "boolean" - }, - "supports_parallel_function_calling": { - "type": "boolean" - }, - "supports_parallel_tool_use_config": { - "type": "boolean" - }, - "supports_pdf_input": { - "type": "boolean" - }, - "supports_prompt_cache_breakpoint": { - "type": "boolean" - }, - "supports_prompt_caching": { - "type": "boolean" - }, - "supports_reasoning": { - "type": "boolean" - }, - "supports_regex_lookaround": { - "type": "boolean" - }, - "supports_response_schema": { - "type": "boolean" - }, - "supports_sampling_params": { - "type": "boolean" - }, - "supports_speed": { - "type": "boolean" - }, - "supports_system_messages": { - "type": "boolean" - }, - "supports_thinking_cache_preservation": { - "type": "boolean" - }, - "supports_tool_choice": { - "type": "boolean" - }, - "supports_tool_search": { - "type": "boolean" - }, - "supports_url_context": { - "type": "boolean" - }, - "supports_video_input": { - "type": "boolean" - }, - "supports_vision": { - "type": "boolean" - }, - "supports_web_search": { - "type": "boolean" - }, - "supports_xhigh_reasoning_effort": { - "type": "boolean" - }, - "thinking_always_on": { - "type": "boolean" - }, - "tiered_pricing": { - "type": "array", - "description": "Context-length or result-count tiered rates; each tier's costs apply within its range.", - "items": { - "type": "object", - "properties": { - "range": { - "type": "array", - "description": "[min, max] prompt-token span this tier applies to.", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 2, - "maxItems": 2 - }, - "max_results_range": { - "type": "array", - "description": "[min, max] result-count span this tier applies to (search models).", - "items": { - "type": "number", - "minimum": 0 - }, - "minItems": 2, - "maxItems": 2 - }, - "input_cost_per_token": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_token": { - "type": "number", - "minimum": 0 - }, - "output_cost_per_reasoning_token": { - "type": "number", - "minimum": 0 - }, - "cache_read_input_token_cost": { - "type": "number", - "minimum": 0 - }, - "cache_creation_input_token_cost": { - "type": "number", - "minimum": 0 - }, - "input_cost_per_query": { - "type": "number", - "minimum": 0 - } - }, - "additionalProperties": false - } - }, - "token_count_framing": { - "type": "integer", - "minimum": 0, - "description": "Tokens added once per request to a local count estimate, for the provider's chat framing." - }, - "token_count_model": { - "type": "object", - "required": [ - "model" - ], - "additionalProperties": false, - "properties": { - "model": { - "type": "string", - "pattern": "\\S" - }, - "region": { - "type": "string", - "pattern": "\\S" - }, - "litellm_provider": { - "type": "string", - "enum": [ - "bedrock", - "bedrock_mantle" - ] - } - }, - "description": "Model that counts tokens for this one, its counting provider, and the AWS region its count API runs in." - }, - "token_count_tokenizer": { - "type": "string", - "description": "Tokenizer litellm counts this model's requests with when the provider cannot count them.", - "enum": [ - "kimi_k3" - ] - }, - "tpm": { - "type": "integer", - "minimum": 0, - "description": "Provider default tokens-per-minute limit." - }, - "use_openai_responses_path": { - "type": "boolean" - }, - "uses_embed_content": { - "type": "boolean" - }, - "vertex_ai_audio_api": { - "type": "string", - "enum": [ - "lyria_predict", - "lyria_interactions" - ] - }, - "web_search_billing_unit": { - "type": "string", - "description": "Whether web search is billed per query or per prompt.", - "enum": [ - "per_query", - "per_prompt" - ] - } - }, - "additionalProperties": true - } - } -} diff --git a/tests/unit/test_circleci_path_filter.py b/tests/unit/test_circleci_path_filter.py index f94b2fa863a..c44560b96f1 100644 --- a/tests/unit/test_circleci_path_filter.py +++ b/tests/unit/test_circleci_path_filter.py @@ -96,7 +96,6 @@ CI = [".github/workflows/test-unit.yml"] ("ui", BACKEND + CLIENT, "run"), ("cost-map-only", ["model_prices_and_context_window.json"], "run"), ("cost-map-only", ["litellm/model_prices_and_context_window_backup.json"], "run"), - ("cost-map-only", ["model_prices_and_context_window.schema.json"], "run"), ( "cost-map-only", ["model_prices_and_context_window.json", "tests/unit/proxy/test_y.py"], diff --git a/tests/unit/test_cost_map_guard.py b/tests/unit/test_cost_map_guard.py index a705597952b..578d5482822 100644 --- a/tests/unit/test_cost_map_guard.py +++ b/tests/unit/test_cost_map_guard.py @@ -186,13 +186,6 @@ def test_bot_may_only_touch_the_cost_map_files() -> None: ) -def test_bot_may_update_cost_map_and_legacy_schema() -> None: - head: Final = _snapshot({**BASE_MAP, "openrouter/c": _entry()}) - changed: Final = (*guard.GUARDED_PATHS, guard.LEGACY_SCHEMA_PATH) - - assert _failures(head, changed_files=changed) == () - - def test_bot_may_not_remove_models() -> None: head = _snapshot({key: value for key, value in BASE_MAP.items() if key != "openrouter/b"}) assert _failures(head, bot=False) == () From cc156a73b8603df1f166bced0ba28c9370fecc60 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 21 Oct 2026 01:58:25 +0000 Subject: [PATCH 039/235] fix(fine_tuning): stop forwarding api_version to the OpenAI client (#44995) The OpenAI fine-tuning handler built its SDK client from every local it received, so an api_version or litellm_params set on an openai finetune_settings entry reached OpenAI.__init__ and every fine-tuning call on that provider failed with "unexpected keyword argument 'api_version'". Build the client from its named params instead and cover it with a request-level regression test Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/unit/fine_tuning/test_main.py | 42 ++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/tests/unit/fine_tuning/test_main.py b/tests/unit/fine_tuning/test_main.py index 57c845c6e1a..89f51d2e74a 100644 --- a/tests/unit/fine_tuning/test_main.py +++ b/tests/unit/fine_tuning/test_main.py @@ -1,11 +1,12 @@ import json -from typing import Final +from typing import Final, Literal import httpx import pytest import respx import litellm +from litellm.types.utils import LiteLLMFineTuningJob API_BASE: Final = "https://fine-tuning.test" BASE_MODEL: Final = "gpt-4.1-nano-2025-04-14" @@ -52,3 +53,42 @@ async def test_create_sends_the_model_without_the_provider_prefix( ) assert json.loads(sent.calls.last.request.content)["model"] == BASE_MODEL + + +@pytest.mark.asyncio +@respx.mock +@pytest.mark.parametrize( + ("custom_llm_provider", "api_base", "route", "sent_api_version"), + [ + ("openai", f"{API_BASE}/v1", f"{API_BASE}/v1/fine_tuning/jobs", None), + ("azure", API_BASE, f"{API_BASE}/openai/fine_tuning/jobs", "2024-10-21"), + ], +) +@pytest.mark.parametrize("is_async", [True, False], ids=["async", "sync"]) +async def test_create_with_api_version_reaches_the_provider_and_only_azure_gets_it( + monkeypatch: pytest.MonkeyPatch, + custom_llm_provider: Literal["openai", "azure"], + api_base: str, + route: str, + sent_api_version: str | None, + is_async: bool, +) -> None: + sent: Final = respx.post(url__startswith=route).mock(return_value=httpx.Response(200, json=PROVIDER_JOB)) + request: Final = { + "model": BASE_MODEL, + "training_file": "file-abc123", + "custom_llm_provider": custom_llm_provider, + "api_key": "sk-test", + "api_base": api_base, + "api_version": "2024-10-21", + } + + async with httpx.AsyncClient() as session: + monkeypatch.setattr(litellm, "aclient_session", session) + job: Final = ( + await litellm.acreate_fine_tuning_job(**request) if is_async else litellm.create_fine_tuning_job(**request) + ) + + assert isinstance(job, LiteLLMFineTuningJob) + assert job.id == PROVIDER_JOB["id"] + assert sent.calls.last.request.url.params.get("api-version") == sent_api_version From 5e189c7267ba60050de6e717b40af4f591cf3d8e Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 19:12:29 -0700 Subject: [PATCH 040/235] test(perf): allowlist anyio TaskHandle._run_coro in event_loop_stalls (#47746) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/cases/event_loop_stalls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/e2e/perf/cases/event_loop_stalls.py b/tests/e2e/perf/cases/event_loop_stalls.py index 0d67a98257f..8a5654cbf7b 100644 --- a/tests/e2e/perf/cases/event_loop_stalls.py +++ b/tests/e2e/perf/cases/event_loop_stalls.py @@ -8,8 +8,8 @@ new hook that deep-copies the request inline all land here as a new row. Runs on `PYTHONASYNCIODEBUG=1` and `LITELLM_EVENT_LOOP=asyncio` (uvloop reports nothing and crashed under debug locally), and the runner `E2E_PERF_EVENT_LOOP=1`, so the nightly build, which sets neither, skips it. The allowlist counts are twice the highest count any of ten back-to-back local runs showed on that row (rows that showed nothing in -those ten but did in the earlier before/after legs for 46925 and 46967 get 2), so each run sits at or under half -of every cap and a listed location doubling in frequency is a failure, not a flake. The +those ten but did in the earlier before/after legs for 46925 and 46967, or in the first scheduled-lane run, get 2), +so each run sits at or under half of every cap and a listed location doubling in frequency is a failure, not a flake. The `garbage collection` bucket is the gen-2 passes the collector logs in debug mode (0.6 to 0.7 s each, one every minute or two of this traffic, plus the ramp while the heap fills on a local whole-file run), which asyncio would otherwise charge to a random task; its count is the signal for a change that makes much more garbage.""" @@ -44,6 +44,7 @@ CASE: Final = replace( KnownStall("run_coroutine_job", max_count=2), KnownStall("user_api_key_auth.py", max_count=2), KnownStall("route_streaming_logging_to_handler", max_count=2), + KnownStall("TaskHandle._run_coro", max_count=2), ), ), ) From f768fb32ef8abfb87c8fc19f1ef4c33dfac3e2b9 Mon Sep 17 00:00:00 2001 From: alberto Date: Wed, 21 Oct 2026 03:26:05 +0100 Subject: [PATCH 041/235] fix(streaming)!: send the include_usage chunk with an empty choices array (#47109) * fix: Send the include_usage chunk with an empty choices array Signed-off-by: Alberto Perdomo * test: Skip chunks with no choices when reading streams Signed-off-by: Alberto Perdomo * fix: Strip usage from plain-iterator async stream chunks Signed-off-by: Alberto Perdomo * test: Expect empty choices on the usage chunk in the gemma tracing test Signed-off-by: Alberto Perdomo * fix(streaming): send the empty-choices usage chunk only to callers that asked for include_usage The usage chunk now has choices=[] only when the caller set stream_options.include_usage itself. When the OpenAI handler, the proxy's always_include_stream_usage setting, or the harness added include_usage on the caller's behalf, the chunk keeps its legacy single empty-delta choice, so clients that read choices[0] on every chunk keep working. The proxy and harness mark their injection with the private _litellm_stream_usage_injected flag, and the proxy scrubs a client-sent copy. Plain-iterator streams keep usage on their chunk when usage was not requested, and the LLM Shield trailing flush ignores chunks with no choices. * fix(proxy): mark the default stream usage injection so hooks keep the legacy usage chunk shape The proxy's default strip-mode injection adds include_usage for clients that did not ask and drops the usage chunk before it reaches them, but streaming iterator hooks still saw it with choices=[]. Marking that injection with _litellm_stream_usage_injected keeps the old single empty choice for them. * fix(router): treat deployment and router-default stream_options as injected, and drop the client flag on the queue route * refactor(router): move the stream usage injection check into router_utils * fix(router): read stream_options from pydantic deployment params when marking injected usage Update the mantle, openinference, and text-completion integration tests to expect the include_usage chunk with empty choices. * fix(proxy): keep the stream usage injection marker out of stored request bodies * test(streaming): cover who asked for stream usage on chat, queue, deployment and spend log paths * test: type the queue flag test, pin the logging start time, and build the deployment usage config without mutation --------- Signed-off-by: Alberto Perdomo Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/harness/endpoint.py | 18 ++- litellm/litellm_core_utils/litellm_logging.py | 2 + .../litellm_core_utils/streaming_handler.py | 64 ++++++---- litellm/proxy/common_request_processing.py | 15 ++- .../llm_shield_proxy/llm_shield_proxy.py | 5 +- litellm/proxy/litellm_pre_call_utils.py | 12 +- litellm/proxy/proxy_server.py | 2 + litellm/router.py | 19 ++- .../router_utils/stream_usage_injection.py | 17 +++ litellm/types/litellm_params.py | 2 + litellm/utils.py | 16 ++- .../compatibility/test_openai_consumer.py | 1 - .../observability/_openinference_support.py | 2 +- .../test_bedrock_mantle_claude_chat_wire.py | 41 +++--- .../test_spend_log_tool_payload_content.py | 29 ++++- .../streaming/test_stream_contracts.py | 120 +++++++++++++++++- tests/unit/harness/test_endpoint.py | 1 + .../test_streaming_handler.py | 85 +++++++++++++ .../completion/test_completion_handler.py | 3 +- .../test_sagemaker_nova_transformation.py | 5 +- ...test_vertex_and_google_ai_studio_gemini.py | 4 +- .../test_vertex_gemma_transformation.py | 2 +- .../guardrail_hooks/test_llm_shield_proxy.py | 15 +++ .../test_pass_through_endpoints.py | 1 + .../proxy/proxy_server/test_routes_queue.py | 22 ++++ .../proxy/test_common_request_processing.py | 33 +++++ .../unit/proxy/test_litellm_pre_call_utils.py | 20 +++ tests/unit/test_main.py | 6 +- tests/unit/test_router/test_router.py | 67 ++++++++++ tests/unit/types/test_litellm_params.py | 1 + 30 files changed, 550 insertions(+), 80 deletions(-) create mode 100644 litellm/router_utils/stream_usage_injection.py diff --git a/litellm/harness/endpoint.py b/litellm/harness/endpoint.py index 0abf7c11280..71e2742fa2e 100644 --- a/litellm/harness/endpoint.py +++ b/litellm/harness/endpoint.py @@ -37,6 +37,7 @@ from litellm.harness.context import GatewayTarget from litellm.harness.errors import HarnessError, HarnessInstallFailed from litellm.harness.types import Harness, Usage from litellm.llms.custom_httpx.http_handler import get_async_httpx_client +from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.llms.custom_http import httpxSpecialProvider if TYPE_CHECKING: @@ -690,18 +691,21 @@ class ModelEndpoint: async def _invoke_sdk( self, route: str, - kwargs: dict[str, Any], # mutable-ok: injects stream_options into the SDK kwargs + kwargs: dict[str, Any], # mutable-ok: SDK call kwargs built by _sdk_kwargs ) -> object: if route == ROUTE_MESSAGES: return await litellm.anthropic.messages.acreate(**kwargs) if route == ROUTE_CHAT: - if kwargs.get("stream"): - stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field - kwargs["stream_options"] = { # mutable-ok: JSON field sent to litellm.acompletion - "include_usage": True, - **stream_options, + if not kwargs.get("stream"): + return await litellm.acompletion(**kwargs) + stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field + return await litellm.acompletion( + **{ + **kwargs, + "stream_options": {"include_usage": True, **stream_options}, + STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options, } - return await litellm.acompletion(**kwargs) + ) return await litellm.aresponses(**kwargs) async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response: diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 473666e5ba1..45349bb0e5a 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -138,6 +138,7 @@ from litellm.types.interactions import ( InteractionsAPIResponse, InteractionsAPIStreamingResponse, ) +from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.llms.openai import ( AllMessageValues, Batch, @@ -698,6 +699,7 @@ class Logging(LiteLLMLoggingBaseClass): self._pre_call_session_id: str = session_id_var.get() _sid: Final = kwargs.get("litellm_session_id") if kwargs else None self.litellm_session_id: str = str(_sid) if _sid else "" + self.stream_usage_injected: Final = kwargs is not None and kwargs.get(STREAM_USAGE_INJECTED_FIELD) is True # supports_correlation_logging is False for calls originating from the # sync client entry point (wrapper() in utils.py): a plain OS thread # has no per-call context isolation the way an asyncio Task does, and diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 6ec426d6da5..b210085c5cc 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -342,6 +342,10 @@ class CustomStreamWrapper: self.messages = getattr(logging_obj, "messages", None) self.sent_stream_usage = False self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False + self.caller_requested_stream_usage: Final = ( + self.check_send_stream_usage(getattr(logging_obj, "stream_options", None)) + and getattr(logging_obj, "stream_usage_injected", False) is not True + ) self.tool_call = False # keep track of the returned chunks - used for calculating the input/output tokens for stream options self.chunks: list[ModelResponseStream] = [] @@ -2070,7 +2074,7 @@ class CustomStreamWrapper: except StopIteration: if self.sent_last_chunk is True: complete_streaming_response: Final = self._build_complete_streaming_response() - response = self.model_response_creator() + response = self._new_usage_chunk() if complete_streaming_response is not None: self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) @@ -2222,27 +2226,10 @@ class CustomStreamWrapper: ) if _has_usage: - # Store a copy ONLY when usage stripping below will mutate - # the chunk. For non-usage chunks (vast majority), store - # directly to avoid expensive model_copy() per chunk. - self.chunks.append(processed_chunk.model_copy()) - - # Strip usage from the outgoing chunk so it's not sent twice - # (once in the chunk, once in _hidden_params). - obj_dict = processed_chunk.model_dump() - if "usage" in obj_dict: - del obj_dict["usage"] - processed_chunk = self.model_response_creator( - chunk=obj_dict, - hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage - Mapping[str, object], getattr(processed_chunk, HIDDEN_PARAMS_ATTR) - ), - ) - is_empty = is_model_response_stream_empty( - model_response=cast(ModelResponseStream, processed_chunk) - ) - if is_empty: + usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk) + if usage_free_chunk is None: continue + processed_chunk = usage_free_chunk else: # No usage data — safe to store directly without copying self.chunks.append(processed_chunk) @@ -2273,13 +2260,18 @@ class CustomStreamWrapper: if processed_chunk is None: continue - choice = processed_chunk.choices[0] - if isinstance(choice, StreamingChoices): - self._accumulate_streamed_delta(choice.delta) + if processed_chunk.choices: + choice = processed_chunk.choices[0] + if isinstance(choice, StreamingChoices): + self._accumulate_streamed_delta(choice.delta) self.rules.post_call_rules(input=self.response_uptil_now, model=self.model) - # RETURN RESULT - self.chunks.append(processed_chunk) - return processed_chunk + if not self.send_stream_usage or getattr(processed_chunk, "usage", None) is None: + self.chunks.append(processed_chunk) + return processed_chunk + usage_free_chunk = self._record_and_strip_usage_chunk(processed_chunk) + if usage_free_chunk is None: + continue + return usage_free_chunk except (StopAsyncIteration, StopIteration): return await self._finalize_completed_stream(cache_hit=cache_hit) except httpx.TimeoutException as e: # if httpx read timeout error occues @@ -2306,7 +2298,7 @@ class CustomStreamWrapper: async def _finalize_completed_stream(self, cache_hit: bool) -> "ModelResponseStream": if self.sent_last_chunk is True: complete_streaming_response: Final = await self._async_build_complete_streaming_response() - response: Final = self.model_response_creator() + response: Final = self._new_usage_chunk() if complete_streaming_response is not None: self._propagate_usage_cost_to_hidden_params(complete_streaming_response, self.custom_llm_provider) @@ -2463,6 +2455,22 @@ class CustomStreamWrapper: return False return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS) + def _new_usage_chunk(self) -> ModelResponseStream: + usage_chunk: Final = self.model_response_creator() + if self.caller_requested_stream_usage: + usage_chunk.choices = [] + return usage_chunk + + def _record_and_strip_usage_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None: + self.chunks.append(chunk.model_copy()) + usage_free_chunk: Final = self.model_response_creator( + chunk={key: value for key, value in chunk.model_dump().items() if key != "usage"}, + hidden_params=cast( # cast-ok: preserve mapping operations on dynamic storage + Mapping[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR) + ), + ) + return None if is_model_response_stream_empty(model_response=usage_free_chunk) else usage_free_chunk + def _accumulate_streamed_delta(self, delta: object) -> None: # Shared by the sync, async, and non-aiohttp iteration sites so answer # text and the disqualifying-content latch stay in step across all three. diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 59db23820fc..cf616873a2e 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -265,6 +265,7 @@ from litellm.proxy.litellm_pre_call_utils import ( sanitize_for_log, ) from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval +from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.utils import ( ModelResponse, ModelResponseStream, @@ -551,22 +552,29 @@ def model_deployments_support_stream_options( return all(_litellm_model_supports_stream_options(m) for m in candidate_models) +_STREAM_USAGE_FLAGS: Final = ("_litellm_strip_stream_usage", STREAM_USAGE_INJECTED_FIELD) + + def stream_usage_tracking_updates( data: Mapping[str, object], general_settings: Mapping[str, object], route_type: str, supports_stream_options: Callable[[], bool], ) -> Mapping[str, object]: - scrub: Final = {"_litellm_strip_stream_usage": False} if "_litellm_strip_stream_usage" in data else {} + scrub: Final = {flag: False for flag in _STREAM_USAGE_FLAGS if flag in data} if data.get("stream", False) is not True: return scrub always_include: Final = general_settings.get("always_include_stream_usage") stream_options: Final = data.get("stream_options") if always_include is True: if "stream_options" not in data: - return {**scrub, "stream_options": {"include_usage": True}} + return {**scrub, "stream_options": {"include_usage": True}, STREAM_USAGE_INJECTED_FIELD: True} if isinstance(stream_options, dict) and "include_usage" not in stream_options: - return {**scrub, "stream_options": {**stream_options, "include_usage": True}} + return { + **scrub, + "stream_options": {**stream_options, "include_usage": True}, + STREAM_USAGE_INJECTED_FIELD: True, + } return scrub if always_include is False and route_type == "agenerate_content_stream" and "stream_options" not in data: return {**scrub, "stream_options": {"include_usage": False}} @@ -580,6 +588,7 @@ def stream_usage_tracking_updates( return { "stream_options": {**merged_stream_options, "include_usage": True}, "_litellm_strip_stream_usage": True, + STREAM_USAGE_INJECTED_FIELD: True, } diff --git a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py index 06ea3ce4b20..7fb87080cd1 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py +++ b/litellm/proxy/guardrails/guardrail_hooks/llm_shield_proxy/llm_shield_proxy.py @@ -494,7 +494,7 @@ class LLMShieldProxyGuardrail(CustomGuardrail): sse: Final = AnthropicSSERestorer(step) events: Final = ResponsesStreamRestorer(step, rehydrate) carries: Final[CarryWindows] = {} - last_chunk = None # rebind-ok: tracks the most recent chunk for the final flush. + last_chunk = None # rebind-ok: tracks the most recent chunk with choices for the final flush. async for chunk in response: if isinstance(chunk, (bytes, str)): @@ -506,7 +506,8 @@ class LLMShieldProxyGuardrail(CustomGuardrail): yield event continue restored_chunk = detached(chunk) - last_chunk = restored_chunk + if getattr(restored_chunk, "choices", None): + last_chunk = restored_chunk for choice in attribute_items(restored_chunk, "choices"): await self._restore_choice(choice, carries, session_id) yield restored_chunk diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 09f53cecb72..d34787cd5e9 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -268,7 +268,7 @@ def sanitize_for_log(value: object) -> str: from litellm.router import Router from litellm.secret_managers.main import get_secret_bool -from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES +from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES, STREAM_USAGE_INJECTED_FIELD from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS from litellm.types.router import BaggageMappedTagsStamp, holds_api_key_secret_reference from litellm.types.services import ServiceTypes @@ -2370,7 +2370,15 @@ def refresh_proxy_server_request_body_snapshot( if not isinstance(proxy_server_request, dict): return _body_snapshot_exclude: Final = ( - frozenset({"secret_fields", "proxy_server_request", "litellm_logging_obj", VECTOR_STORE_CALLER_METADATA_KWARG}) + frozenset( + { + "secret_fields", + "proxy_server_request", + "litellm_logging_obj", + VECTOR_STORE_CALLER_METADATA_KWARG, + STREAM_USAGE_INJECTED_FIELD, + } + ) | _TRANSPORT_ONLY_CREDENTIAL_KEYS | _CALLBACK_CREDENTIAL_KEYS ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8d6c9dd925b..b48b5ab1c1e 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -976,6 +976,7 @@ from litellm.secret_managers.main import ( ) from litellm.tracing.config import is_lens_tracing_enabled from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs +from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, AnthropicResponse, @@ -18219,6 +18220,7 @@ async def async_queue_request( try: data = await request.json() data.pop("_litellm_strip_stream_usage", None) + data.pop(STREAM_USAGE_INJECTED_FIELD, None) # Include original request and headers in the data data["proxy_server_request"] = { diff --git a/litellm/router.py b/litellm/router.py index a18ad7c3a97..1ce1e0a5a70 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -361,10 +361,11 @@ from litellm.router_utils.routing_groups import ( validate_routing_strategy, ) from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch +from litellm.router_utils.stream_usage_injection import router_config_requests_stream_usage from litellm.scheduler import FlowItem, Scheduler from litellm.secret_managers.main import str_to_bool from litellm.types.guardrails import GuardrailEventHooks -from litellm.types.litellm_params import RoutingStrategyName +from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD, RoutingStrategyName from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionToolParam, @@ -4673,7 +4674,8 @@ class Router: deployment=deployment, forwarded_keys=kwargs.pop(_ALIAS_MARKER_FORWARDED_PARAMS_KWARG, ()) ): kwargs.pop(key, None) - self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=self.default_litellm_params) + defaults: Final = _LITELLM_PARAMS_ADAPTER.validate_python(self.default_litellm_params) + self._merge_tools_from_deployment(deployment=deployment, kwargs=kwargs, defaults=defaults) model_info = deployment.get("model_info", {}).copy() deployment_litellm_model_name = deployment["litellm_params"]["model"] @@ -4817,6 +4819,19 @@ class Router: data=_DEPLOYMENT_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]), ) + deployment_litellm_params: Final = _DEPLOYMENT_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]) + if router_config_requests_stream_usage( + _REQUEST_KWARGS_ADAPTER.validate_python(kwargs), + ( + _LITELLM_PARAMS_ADAPTER.validate_python( + deployment_litellm_params + if isinstance(deployment_litellm_params, Mapping) + else deployment_litellm_params.model_dump(exclude_none=True) + ), + defaults, + ), + ): + kwargs[STREAM_USAGE_INJECTED_FIELD] = True self._update_kwargs_with_default_litellm_params(kwargs=kwargs, metadata_variable_name=metadata_variable_name) hop_kwargs: Final = _FALLBACK_HOP_ADAPTER.validate_python(kwargs) if _is_fallback_hop(hop_kwargs): diff --git a/litellm/router_utils/stream_usage_injection.py b/litellm/router_utils/stream_usage_injection.py new file mode 100644 index 00000000000..946543d9b83 --- /dev/null +++ b/litellm/router_utils/stream_usage_injection.py @@ -0,0 +1,17 @@ +from collections.abc import Mapping, Sequence + + +def router_config_requests_stream_usage( + request_kwargs: Mapping[str, object], configured_params: Sequence[Mapping[str, object]] +) -> bool: + if request_kwargs.get("stream") is not True or "stream_options" in request_kwargs: + return False + return any(_includes_usage(params.get("stream_options")) for params in configured_params) + + +def _includes_usage(stream_options: object) -> bool: + match stream_options: + case {"include_usage": True}: + return True + case _: + return False diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 5f318cc3c69..0ed156524be 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -54,6 +54,7 @@ RoutingStrategyName: TypeAlias = Literal[ TRUSTED_CALLBACK_VARS_FIELD: Final = "litellm_trusted_callback_vars" ADDRESSED_RESPONSE_ID_FIELD: Final = "_litellm_addressed_response_id" +STREAM_USAGE_INJECTED_FIELD: Final = "_litellm_stream_usage_injected" WIRE_NAME: Final = "wire_name" @@ -397,6 +398,7 @@ class ProxyRequestState: trusted_callback_vars: Mapping[str, str] | None = field(default=None, metadata=wire(TRUSTED_CALLBACK_VARS_FIELD)) addressed_response_id: str | None = field(default=None, metadata=wire(ADDRESSED_RESPONSE_ID_FIELD)) strip_stream_usage: bool | None = field(default=None, metadata=wire("_litellm_strip_stream_usage")) + stream_usage_injected: bool | None = field(default=None, metadata=wire(STREAM_USAGE_INJECTED_FIELD)) client_side_timeout: bool | None = None model_file_id_mapping: Mapping[str, Mapping[str, str]] | None = None diff --git a/litellm/utils.py b/litellm/utils.py index 379eddb3155..1aad9baf06a 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7962,12 +7962,16 @@ class TextCompletionStreamWrapper: text_choices: Final = TextChoices() if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices raise Exception - delta: Final = chunk["choices"][0]["delta"] - text_choices["text"] = delta["content"] - text_choices["reasoning_content"] = delta.get("reasoning_content") - text_choices["index"] = chunk["choices"][0]["index"] - text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"] - response["choices"] = [text_choices] + + if chunk["choices"]: + delta: Final = chunk["choices"][0]["delta"] + text_choices["text"] = delta["content"] + text_choices["reasoning_content"] = delta.get("reasoning_content") + text_choices["index"] = chunk["choices"][0]["index"] + text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"] + response["choices"] = [text_choices] + else: + response["choices"] = [] # only pass usage when stream_options["include_usage"] is True if self.stream_options and self.stream_options.get("include_usage", False) is True: diff --git a/tests/integration/compatibility/test_openai_consumer.py b/tests/integration/compatibility/test_openai_consumer.py index a49f20f4711..6150995723b 100644 --- a/tests/integration/compatibility/test_openai_consumer.py +++ b/tests/integration/compatibility/test_openai_consumer.py @@ -459,7 +459,6 @@ async def test_openai_clients_parse_text_completion_streams_and_choices(gateway: def test_text_completion_usage_chunk_has_empty_choices(gateway: Gateway) -> None: - pytest.skip("BUG: the /v1/completions include_usage chunk carries choices [{index: 0}] instead of []") identity: Final = f"cmpl-usage-{uuid.uuid4().hex}" prompt: Final = f"raw-stream {identity}" with wire_server(_text_provider(identity, {prompt})) as wire, gateway.scenario() as scenario: diff --git a/tests/integration/observability/_openinference_support.py b/tests/integration/observability/_openinference_support.py index b46503703c7..58785c89033 100644 --- a/tests/integration/observability/_openinference_support.py +++ b/tests/integration/observability/_openinference_support.py @@ -312,7 +312,7 @@ def _chat_caller_stream_usage_event(event: dict[str, JsonValue], model: str) -> "object": event["object"], "created": 1, "model": model, - "choices": [{"index": 0, "delta": {}}], + "choices": [], "usage": {**_json_object_value(event["usage"]), "cost": 4.05e-6}, } diff --git a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py index 2badf810b6a..d4f9533fcdc 100644 --- a/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py +++ b/tests/integration/providers/test_bedrock_mantle_claude_chat_wire.py @@ -1,6 +1,7 @@ import json import re from collections.abc import Callable, Mapping +from itertools import chain from typing import Final from uuid import uuid4 @@ -263,13 +264,17 @@ def _assert_chunk_envelope(chunks: tuple[dict[str, JsonValue], ...], model: str, return next(iter(identities)) +def _content_chunks(chunks: tuple[dict[str, JsonValue], ...]) -> tuple[dict[str, JsonValue], ...]: + return tuple(chunk for chunk in chunks if chunk.get("usage") is None) + + def _finish_reasons(chunks: tuple[dict[str, JsonValue], ...]) -> list[JsonValue]: - reasons: Final = (_only_choice(chunk).get("finish_reason") for chunk in chunks) + reasons: Final = (_only_choice(chunk).get("finish_reason") for chunk in _content_chunks(chunks)) return [reason for reason in reasons if reason is not None] def _deltas(chunks: tuple[dict[str, JsonValue], ...]) -> tuple[dict[str, JsonValue], ...]: - return tuple(object_value(_only_choice(chunk)["delta"]) for chunk in chunks) + return tuple(object_value(_only_choice(chunk)["delta"]) for chunk in _content_chunks(chunks)) def _sdk_base(gateway: Gateway) -> str: @@ -301,12 +306,12 @@ def test_chat_completion_on_a_mantle_claude_id_posts_native_messages_and_spends_ @pytest.mark.parametrize( ("stream_options", "usage_chunks"), [ - pytest.param({"stream_options": {"include_usage": True}}, [_USAGE], id="include_usage"), + pytest.param({"stream_options": {"include_usage": True}}, [(True, *_USAGE)], id="include_usage"), pytest.param({}, [], id="no_usage"), ], ) def test_chat_stream_on_a_mantle_claude_id_relays_openai_chunks_from_the_native_stream( - gateway: Gateway, stream_options: dict[str, JsonValue], usage_chunks: list[tuple[int, int, int]] + gateway: Gateway, stream_options: dict[str, JsonValue], usage_chunks: list[tuple[JsonValue, ...]] ) -> None: marker: Final = uuid4().hex expected: Final = _outbound(_HAIKU, [_user_turn(marker)], stream=True) @@ -322,9 +327,9 @@ def test_chat_stream_on_a_mantle_claude_id_relays_openai_chunks_from_the_native_ identity: Final = _assert_chunk_envelope(chunks, model, response.text) assert "".join(str(delta.get("content") or "") for delta in _deltas(chunks)) == _answer(marker), response.text assert _finish_reasons(chunks) == ["stop"], response.text - assert [_token_usage(chunk) for chunk in chunks if chunk.get("usage") is not None] == usage_chunks, ( - response.text - ) + assert [ + (chunk["choices"] == [], *_token_usage(chunk)) for chunk in chunks if chunk.get("usage") is not None + ] == usage_chunks, response.text _assert_success_row( _spend_row(_BY_REQUEST_ID, identity), model, _HAIKU, "acompletion", _mantle_cost(gateway, _HAIKU) ) @@ -620,15 +625,14 @@ def test_openai_sdk_chat_stream_on_a_mantle_claude_id(gateway: Gateway) -> None: ) identities: Final = {chunk.id for chunk in chunks} assert len(identities) == 1 and {chunk.model for chunk in chunks} == {model}, chunks - assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _answer(marker), chunks - assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"], ( - chunks - ) + choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks)) + assert "".join(choice.delta.content or "" for choice in choices) == _answer(marker), chunks + assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["stop"], chunks assert [ - (chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens) + (len(chunk.choices), chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens) for chunk in chunks if chunk.usage is not None - ] == [_USAGE], chunks + ] == [(0, *_USAGE)], chunks _assert_success_row( _spend_row(_BY_REQUEST_ID, next(iter(identities))), model, @@ -688,15 +692,14 @@ async def test_async_openai_sdk_chat_stream_on_a_mantle_claude_id(gateway: Gatew chunks: Final = tuple([chunk async for chunk in stream]) identities: Final = {chunk.id for chunk in chunks} assert len(identities) == 1 and {chunk.model for chunk in chunks} == {model}, chunks - assert "".join(chunk.choices[0].delta.content or "" for chunk in chunks) == _answer(marker), chunks - assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["stop"], ( - chunks - ) + choices: Final = tuple(chain.from_iterable(chunk.choices for chunk in chunks)) + assert "".join(choice.delta.content or "" for choice in choices) == _answer(marker), chunks + assert [choice.finish_reason for choice in choices if choice.finish_reason] == ["stop"], chunks assert [ - (chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens) + (len(chunk.choices), chunk.usage.prompt_tokens, chunk.usage.completion_tokens, chunk.usage.total_tokens) for chunk in chunks if chunk.usage is not None - ] == [_USAGE], chunks + ] == [(0, *_USAGE)], chunks _assert_success_row( _spend_row(_BY_REQUEST_ID, next(iter(identities))), model, diff --git a/tests/integration/spend/test_spend_log_tool_payload_content.py b/tests/integration/spend/test_spend_log_tool_payload_content.py index 9055f6f06c4..226accdbf54 100644 --- a/tests/integration/spend/test_spend_log_tool_payload_content.py +++ b/tests/integration/spend/test_spend_log_tool_payload_content.py @@ -15,7 +15,7 @@ import openai import pytest from integration._support.client import Gateway, eventually, object_value, string_value from integration._support.database import read_rows -from integration._support.process import owned_proxy +from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.wire import Reply, Request, wire_server from pydantic import JsonValue, TypeAdapter @@ -723,6 +723,33 @@ def test_chat_sdk_stream_include_usage_masks_request_fields(gateway: Gateway, tm assert len(post_requests) == 1 +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +def test_chat_stream_usage_the_proxy_adds_keeps_its_marker_out_of_the_stored_request( + gateway: Gateway, tmp_path: Path +) -> None: + response_id: Final = f"chatcmpl-proxy-usage-{uuid4()}" + with ( + wire_server( + _answering_model_listing(lambda request: _chat_stream(response_id, "sort_key", include_usage=True)) + ) as wire, + owned_proxy(gateway, tmp_path, {}, config=_prompt_storage_config(tmp_path)) as isolated, + isolated.scenario() as scenario, + ): + model: Final = scenario.model(model="openai/gpt-4o-mini", api_base=wire.url, api_key="synthetic-openai-key") + response: Final = isolated.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "stream marker"}], "stream": True}, + ) + assert response.status_code == 200, response.text + assert {string_value(event["id"]) for event in _sse_events(response.text)} == {response_id}, response.text + stored_request: Final = object_value(_stored_row(response_id)["proxy_server_request"]) + assert stored_request["messages"] == [{"role": "user", "content": "stream marker"}], stored_request + assert "_litellm_stream_usage_injected" not in stored_request, stored_request + observed: Final = _provider_calls(wire.drain()) + assert [_json_object(request.body)["stream_options"] for request in observed] == [{"include_usage": True}] + + def test_chat_history_keeps_string_tool_arguments_and_tool_content(gateway: Gateway, tmp_path: Path) -> None: arguments: Final = '{"sort_key":"created_at"}' tool_content: Final = "tool-result-created_at" diff --git a/tests/integration/streaming/test_stream_contracts.py b/tests/integration/streaming/test_stream_contracts.py index d7537a78db8..1d2a8b25792 100644 --- a/tests/integration/streaming/test_stream_contracts.py +++ b/tests/integration/streaming/test_stream_contracts.py @@ -3,6 +3,7 @@ import json import threading import uuid from collections.abc import Callable +from itertools import chain from pathlib import Path from queue import SimpleQueue from typing import Final @@ -13,9 +14,10 @@ from hypothesis import Phase, example, given, settings from hypothesis import strategies as st from integration._support.client import Gateway, eventually, object_value, string_value from integration._support.database import read_rows -from integration._support.process import owned_proxy +from integration._support.process import graceful_stop_seconds, owned_proxy from integration._support.wire import Reply, Request, wire_server -from openai import OpenAI +from openai import OpenAI, omit +from openai.types.chat import ChatCompletionChunk from pydantic import JsonValue, TypeAdapter pytestmark: Final = pytest.mark.shares_worker_state @@ -197,6 +199,7 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat assert {chunk.id for chunk in chunks} == {identity} usages: Final = tuple(chunk.usage for chunk in chunks if chunk.usage is not None) assert len(usages) == (1 if include else 0) + assert [len(chunk.choices) for chunk in chunks if chunk.usage is not None] == ([0] if include else []) if include: assert usages[0].prompt_tokens == 11 and usages[0].completion_tokens == 4 requests: Final = wire.drain() @@ -214,6 +217,119 @@ def test_proxy_stream_usage_visibility_keeps_exact_persisted_charge(gateway: Gat assert float(rows[0]["spend"]) == pytest.approx(0.019) +_STREAM_USAGE_INJECTED_FLAG: Final = "_litellm_stream_usage_injected" + + +def _registered_model_names(gateway: Gateway) -> tuple[JsonValue, ...]: + entries: Final = gateway.get("/model/info")["data"] + assert isinstance(entries, list), entries + return tuple(object_value(entry).get("model_name") for entry in entries) + + +def _registered(gateway: Gateway, model: str) -> None: + eventually(lambda: _registered_model_names(gateway), lambda model_names: model in model_names, seconds=60) + + +def _sdk_stream( + gateway: Gateway, + route_prefix: str, + model: str, + identity: str, + *, + include_usage: bool | None, + extra_body: dict[str, JsonValue], +) -> tuple[ChatCompletionChunk, ...]: + with OpenAI( + api_key=gateway.key, + base_url=str(gateway.client.base_url).rstrip("/") + route_prefix, + timeout=30, + max_retries=0, + ) as client: + stream: Final = client.chat.completions.create( + model=model, + messages=[{"role": "user", "content": identity}], + stream=True, + stream_options=omit if include_usage is None else {"include_usage": include_usage}, + extra_body=extra_body, + ) + with stream: + return tuple(stream) + + +def _streamed_text(chunks: tuple[ChatCompletionChunk, ...]) -> str: + choices: Final = chain.from_iterable(chunk.choices for chunk in chunks) + return "".join(choice.delta.content or "" for choice in choices) + + +def _usage_chunk_shapes(chunks: tuple[ChatCompletionChunk, ...]) -> list[tuple[int, int]]: + return [(len(chunk.choices), chunk.usage.total_tokens) for chunk in chunks if chunk.usage is not None] + + +@pytest.mark.parametrize( + ("route_prefix", "sends", "extra_body"), + [pytest.param("/v1", 2, {}, id="chat_miss_then_cache_hit"), pytest.param("/queue", 1, {"priority": 0}, id="queue")], +) +def test_caller_requested_stream_usage_has_no_choices_even_with_a_client_sent_injection_flag( + gateway: Gateway, route_prefix: str, sends: int, extra_body: dict[str, JsonValue] +) -> None: + identity: Final = "stream-usage-flag-" + uuid.uuid4().hex + with ( + wire_server(lambda request: Reply(content_type="text/event-stream", chunks=text_stream(identity))) as wire, + gateway.scenario() as scenario, + ): + model: Final = scenario.model(api_base=wire.url + "/v1") + _registered(gateway, model) + streams: Final = tuple( + _sdk_stream( + gateway, + route_prefix, + model, + identity, + include_usage=True, + extra_body={**extra_body, _STREAM_USAGE_INJECTED_FLAG: True}, + ) + for _ in range(sends) + ) + assert [_streamed_text(chunks) for chunks in streams] == ["Hello 雪 café"] * sends + assert [_usage_chunk_shapes(chunks) for chunks in streams] == [[(0, 15)]] * sends + requests: Final = wire.drain() + assert len(requests) == 1 + assert _STREAM_USAGE_INJECTED_FLAG not in json.loads(requests[0].body) + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 120) +def test_stream_usage_the_deployment_adds_keeps_one_empty_choice(gateway: Gateway, tmp_path: Path) -> None: + identity: Final = "deployment-stream-usage-" + uuid.uuid4().hex + with wire_server(lambda request: Reply(content_type="text/event-stream", chunks=text_stream(identity))) as wire: + shared_config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config: Final = { + **shared_config, + "model_list": [ + { + "model_name": "deployment-usage", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "synthetic-deployment-usage-key", + "api_base": wire.url + "/v1", + "stream_options": {"include_usage": True}, + }, + } + ], + "general_settings": {**shared_config["general_settings"], "always_include_stream_usage": False}, + } + path: Final = tmp_path / "deployment-stream-usage.yaml" + path.write_text(yaml.safe_dump(config)) + with owned_proxy(gateway, tmp_path, {}, config=path) as candidate: + chunks: Final = _sdk_stream( + candidate, "/v1", "deployment-usage", identity, include_usage=None, extra_body={} + ) + assert _streamed_text(chunks) == "Hello 雪 café" + assert _usage_chunk_shapes(chunks) == [(1, 15)] + requests: Final = wire.drain() + assert len(requests) == 1 + assert json.loads(requests[0].body)["stream_options"]["include_usage"] is True + + @pytest.mark.covers("other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream") def test_messages_stream_completes_through_trailing_empty_choices_usage_chunk(gateway: Gateway) -> None: identity: Final = "messages-empty-choices-" + uuid.uuid4().hex diff --git a/tests/unit/harness/test_endpoint.py b/tests/unit/harness/test_endpoint.py index 19bd6d0450e..d1447fd40cc 100644 --- a/tests/unit/harness/test_endpoint.py +++ b/tests/unit/harness/test_endpoint.py @@ -272,6 +272,7 @@ async def test_sdk_chat_stream(monkeypatch: pytest.MonkeyPatch) -> None: assert lines[-1] == "data: [DONE]" assert json.loads(lines[0][6:])["choices"][0]["delta"]["content"] == "he" assert calls[0]["stream_options"] == {"include_usage": True} + assert calls[0]["_litellm_stream_usage_injected"] is True assert (ep.usage.input_tokens, ep.usage.output_tokens) == (6, 2) assert ep.usage.cost == pytest.approx(6 * 0.001 + 2 * 0.002) diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 68aabc3c01e..550ecd3c0f7 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -603,6 +603,91 @@ async def test_streaming_handler_with_usage( assert chunk_has_usage +def _usage_test_logging( + caller_stream_options: dict[str, bool] | None, stream_usage_injected: bool = False +) -> Logging: + logging_obj: Final = Logging( + model="hosted_vllm/m", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="completion", + start_time=1_790_000_000.0, + litellm_call_id="12345", + function_id="1245", + kwargs={"_litellm_stream_usage_injected": True} if stream_usage_injected else {}, + ) + logging_obj.update_environment_variables( + litellm_params={}, optional_params={}, model="hosted_vllm/m", stream_options=caller_stream_options + ) + return logging_obj + + +@pytest.mark.parametrize( + ("caller_stream_options", "stream_usage_injected", "usage_chunk_choice_count"), + [ + ({"include_usage": True}, False, 0), + (None, False, 1), + ({"include_usage": True}, True, 1), + ], + ids=["caller-asked", "provider-handler-injected", "proxy-injected"], +) +@pytest.mark.parametrize("plain_iterator", [False, True]) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_usage_chunk_has_empty_choices_only_for_callers_that_asked( + sync_mode: bool, + plain_iterator: bool, + caller_stream_options: dict[str, bool] | None, + stream_usage_injected: bool, + usage_chunk_choice_count: int, +): + usage: Final = Usage(prompt_tokens=5, completion_tokens=1, total_tokens=6) + provider_chunks: Final = [ + ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="ok"))]), + ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")]), + ModelResponseStream(id="c1", choices=[], provider_specific_fields={"citations": ["a"]}), + ModelResponseStream(id="c1", choices=[], usage=usage), + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=( + iter(provider_chunks) if plain_iterator else ModelResponseListIterator(model_responses=provider_chunks) + ), + model="hosted_vllm/m", + custom_llm_provider="hosted_vllm", + logging_obj=_usage_test_logging(caller_stream_options, stream_usage_injected), + stream_options={"include_usage": True}, + ) + + chunks: Final = list(wrapper) if sync_mode else [chunk async for chunk in wrapper] + + assert [len(chunk.choices) for chunk in chunks] == [1, 1, 0, usage_chunk_choice_count] + assert chunks[2].provider_specific_fields == {"citations": ["a"]} + assert [getattr(chunk, "usage", None) is not None for chunk in chunks] == [False, False, False, True] + assert chunks[-1].usage.total_tokens == 6 + + +@pytest.mark.asyncio +async def test_plain_iterator_stream_keeps_usage_on_its_chunk_when_usage_was_not_requested(): + usage: Final = Usage(prompt_tokens=5, completion_tokens=1, total_tokens=6) + provider_chunks: Final = [ + ModelResponseStream(id="c1", choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content="ok"))]), + ModelResponseStream( + id="c1", choices=[StreamingChoices(index=0, delta=Delta(), finish_reason="stop")], usage=usage + ), + ] + wrapper: Final = CustomStreamWrapper( + completion_stream=iter(provider_chunks), + model="hosted_vllm/m", + custom_llm_provider="hosted_vllm", + logging_obj=_usage_test_logging(None), + ) + + chunks: Final = [chunk async for chunk in wrapper] + + assert [chunk.choices[0].finish_reason for chunk in chunks] == [None, "stop"] + assert chunks[-1].usage.total_tokens == 6 + + @pytest.mark.parametrize("sync_mode", [False]) @pytest.mark.asyncio @pytest.mark.flaky(reruns=3) diff --git a/tests/unit/llms/openai/completion/test_completion_handler.py b/tests/unit/llms/openai/completion/test_completion_handler.py index f465bbea663..9e2754eef68 100644 --- a/tests/unit/llms/openai/completion/test_completion_handler.py +++ b/tests/unit/llms/openai/completion/test_completion_handler.py @@ -364,7 +364,8 @@ def test_text_completion_stream_forwards_stream_options_and_usage(monkeypatch): ) ) - assert tuple(chunk.choices[0].text for chunk in chunks) == ("hello", None, None) + assert tuple(chunk.choices[0].text for chunk in chunks[:-1]) == ("hello", None) + assert chunks[-1].choices == [] assert tuple(chunk.usage for chunk in chunks[:-1]) == (None, None) assert (chunks[-1].usage.prompt_tokens, chunks[-1].usage.completion_tokens, chunks[-1].usage.total_tokens) == ( 2, diff --git a/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py b/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py index 12690d5822b..56c0aee3efd 100644 --- a/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py +++ b/tests/unit/llms/sagemaker/test_sagemaker_nova_transformation.py @@ -543,8 +543,9 @@ def test_nova_streaming_sends_stream_flag_and_parses_chunks_and_usage(respx_mock "stream": True, "stream_options": {"include_usage": True}, } - assert tuple(chunk.choices[0].delta.content for chunk in chunks) == ("Nova ", "stream", None, None) - assert tuple(chunk.choices[0].finish_reason for chunk in chunks) == (None, None, "stop", None) + assert tuple(chunk.choices[0].delta.content for chunk in chunks[:-1]) == ("Nova ", "stream", None) + assert chunks[-1].choices == [] + assert tuple(chunk.choices[0].finish_reason for chunk in chunks[:-1]) == (None, None, "stop") assert (chunks[-1].usage.prompt_tokens, chunks[-1].usage.completion_tokens, chunks[-1].usage.total_tokens) == ( 2, 1, diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index bb624add673..6ea3039edd8 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1894,7 +1894,9 @@ def test_gemini_tool_use(respx_mock: respx.MockRouter) -> None: assert [(call.function.name, json.loads(call.function.arguments)) for call in tool_calls] == [ ("get_weather", {"location": "Lima, Peru"}) ] - assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices[0].finish_reason] == ["tool_calls"] + assert [chunk.choices[0].finish_reason for chunk in chunks if chunk.choices and chunk.choices[0].finish_reason] == [ + "tool_calls" + ] @pytest.mark.respx(assert_all_called=True) diff --git a/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py b/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py index fb1cf4ced2d..0726ebb5ee1 100644 --- a/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_gemma_models/test_vertex_gemma_transformation.py @@ -632,7 +632,7 @@ class TestVertexGemmaCompletion: assert chunks[0].choices[0].delta.content == "READY" assert chunks[1].choices[0].finish_reason == "stop" if stream_options and stream_options["include_usage"]: - assert chunks[-1].choices[0].delta.content is None + assert chunks[-1].choices == [] assert chunks[-1].usage.total_tokens == 114 assert span.get_metric("_dd.llmobs.total_tokens") == 114 else: diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py index 63fcdfbb0c5..df2fc8e1da2 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_llm_shield_proxy.py @@ -1,6 +1,7 @@ import asyncio import json from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock import pytest @@ -1975,6 +1976,20 @@ class TestProxyWiring: class TestStreamUsage: + @pytest.mark.asyncio + async def test_trailing_flush_survives_a_usage_chunk_with_no_choices(self): + guardrail, _ = _shielded({"[EMAIL_1]": "a@example.com"}) + held: Final = ModelResponseStream(choices=[StreamingChoices(index=0, delta=Delta(content="Mail [EMAI"))]) + usage_chunk: Final = ModelResponseStream( + choices=[], usage=Usage(prompt_tokens=5, completion_tokens=7, total_tokens=12) + ) + + out: Final = await _restore_stream(guardrail, [held, usage_chunk]) + + assert out[1] is not None and out[1].choices == [] + assert [chunk.choices[0].delta.content for chunk in out[2:]] == ["[EMAI"] + assert all(getattr(chunk, "usage", None) is None for chunk in out[2:]) + @pytest.mark.asyncio async def test_trailing_flush_does_not_repeat_usage(self): """With n>=2 and include_usage, the last chunk carries usage; a flush copied from it must not. diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index f0baceecdaa..db0a74d360a 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -8536,6 +8536,7 @@ PROXY_STAMPED_NAMES: Final = frozenset( "litellm_trusted_callback_vars", "_litellm_addressed_response_id", "_litellm_strip_stream_usage", + "_litellm_stream_usage_injected", "client_side_timeout", "model_file_id_mapping", ) diff --git a/tests/unit/proxy/proxy_server/test_routes_queue.py b/tests/unit/proxy/proxy_server/test_routes_queue.py index 07a61c8587b..a86314e3063 100644 --- a/tests/unit/proxy/proxy_server/test_routes_queue.py +++ b/tests/unit/proxy/proxy_server/test_routes_queue.py @@ -6,10 +6,13 @@ Pins (PR2): from __future__ import annotations +from collections.abc import Callable +from contextlib import AbstractContextManager from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +from fastapi.testclient import TestClient from litellm import Router from litellm.proxy import proxy_server @@ -85,6 +88,25 @@ def test_queue_chat_completions_happy(client, auth_as, patched_queue): } +def test_queue_chat_completions_drops_client_sent_stream_usage_flags( + client: TestClient, auth_as: Callable[..., AbstractContextManager[None]], patched_queue: MagicMock +) -> None: + payload: Final = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "priority": 0, + "_litellm_strip_stream_usage": True, + "_litellm_stream_usage_injected": True, + } + with auth_as(): + response: Final = client.post("/queue/chat/completions", json=payload) + assert response.status_code == 200 + forwarded: Final = patched_queue.schedule_acompletion.call_args.kwargs + assert "_litellm_strip_stream_usage" not in forwarded + assert "_litellm_stream_usage_injected" not in forwarded + assert forwarded["messages"] == [{"role": "user", "content": "hi"}] + + def test_queue_chat_completions_no_router_error(client, auth_as, queue_no_router): """Pins ``POST /queue/chat/completions`` (error: no llm_router).""" payload = {"model": "gpt-4", "messages": [{"role": "user", "content": "hi"}]} diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index d742956bb59..a226ce8a4a9 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -9181,6 +9181,7 @@ class TestApplyStreamUsageTracking: assert data["stream_options"] == {"include_usage": True} assert data["_litellm_strip_stream_usage"] is True + assert data["_litellm_stream_usage_injected"] is True def test_default_preserves_other_client_stream_options_keys(self): data = {"stream": True, "stream_options": {"include_obfuscation": True}} @@ -9189,6 +9190,7 @@ class TestApplyStreamUsageTracking: assert data["stream_options"] == {"include_obfuscation": True, "include_usage": True} assert data["_litellm_strip_stream_usage"] is True + assert data["_litellm_stream_usage_injected"] is True def test_client_requested_usage_is_left_untouched_and_not_stripped(self): data = {"stream": True, "stream_options": {"include_usage": True}} @@ -9205,6 +9207,7 @@ class TestApplyStreamUsageTracking: assert data["stream_options"]["include_usage"] is True assert data["_litellm_strip_stream_usage"] is True + assert data["_litellm_stream_usage_injected"] is True def test_explicit_false_flag_disables_injection_entirely(self): data = {"stream": True} @@ -9230,6 +9233,36 @@ class TestApplyStreamUsageTracking: assert data["stream_options"] == {"include_usage": True} assert "_litellm_strip_stream_usage" not in data + @pytest.mark.parametrize("client_stream_options", [None, {"include_obfuscation": True}]) + def test_flag_true_marks_usage_injected_for_clients_that_did_not_ask( + self, client_stream_options: dict[str, bool] | None + ): + data: Final = { + "stream": True, + **({} if client_stream_options is None else {"stream_options": client_stream_options}), + } + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": True}, + route_type="acompletion", + ) + + assert data["stream_options"]["include_usage"] is True + assert data["_litellm_stream_usage_injected"] is True + + def test_flag_true_leaves_usage_unmarked_when_client_asked_and_neutralizes_client_marker(self): + data: Final = {"stream": True, "stream_options": {"include_usage": True}, "_litellm_stream_usage_injected": True} + + _apply_stream_usage_tracking( + data=data, + general_settings={"always_include_stream_usage": True}, + route_type="acompletion", + ) + + assert data["stream_options"] == {"include_usage": True} + assert data["_litellm_stream_usage_injected"] is False + def test_flag_true_respects_client_explicit_include_usage_false(self): data = {"stream": True, "stream_options": {"include_usage": False}} diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index c2a5b01fe4e..3f8e3bc93d1 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -936,6 +936,26 @@ def test_body_snapshot_excludes_team_callback_credentials() -> None: }, proxy_request +def test_body_snapshot_keeps_the_stream_usage_injection_marker_out_of_the_stored_request() -> None: + from litellm.proxy.common_request_processing import stream_usage_tracking_updates + from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot + from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD + + request: Final = {"model": "gpt-5-mini", "messages": [{"role": "user", "content": "hi"}], "stream": True} + proxy_request: Final = {"body": dict(request)} + data: Final = { + **request, + **stream_usage_tracking_updates(request, {}, "acompletion", lambda: True), + "proxy_server_request": proxy_request, + } + + refresh_proxy_server_request_body_snapshot(data) + + assert data[STREAM_USAGE_INJECTED_FIELD] is True + assert proxy_request["body"]["stream_options"] == {"include_usage": True} + assert STREAM_USAGE_INJECTED_FIELD not in proxy_request["body"] + + @pytest.mark.asyncio @pytest.mark.parametrize("pre_call_ran", [False, True]) async def test_post_guardrail_snapshot_preserves_logging_only_masking_in_spend_logs( diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index f5d13aecb6f..c3d3575b8b3 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -3868,6 +3868,9 @@ def test_stream_chunk_builder_text_completion_combines_text_and_usage(): model="gpt-3.5-turbo-instruct", choices=[{"text": " world", "index": 0, "logprobs": None, "finish_reason": "stop"}], ), + TextCompletionResponse( + id="cmpl-1", object="text_completion", created=1, model="gpt-3.5-turbo-instruct", choices=[] + ), ] response = stream_chunk_builder_text_completion(chunks=chunks, messages=[{"role": "user", "content": "say hello"}]) @@ -8853,7 +8856,8 @@ async def test_text_completion_stream_include_usage_returns_usage_on_final_chunk "stream": True, "stream_options": {"include_usage": True}, } - assert "".join(chunk.choices[0].text or "" for chunk in chunks) == "reply" + assert "".join(chunk.choices[0].text or "" for chunk in chunks if chunk.choices) == "reply" + assert chunks[-1].choices == [] final_usage: Final = chunks[-1].usage assert (final_usage.prompt_tokens, final_usage.completion_tokens, final_usage.total_tokens) == (3, 2, 5) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 13ac55fa937..b119d216542 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -29670,6 +29670,73 @@ def test_pre_call_checks_ignores_an_image_response_format_string(monkeypatch: py assert [d["model_info"]["id"] for d in filtered] == ["only"] +@pytest.mark.parametrize( + ("deployment_stream_options", "default_stream_options", "request_stream_options", "usage_chunk_choice_count"), + [ + ({"include_usage": True}, None, None, 1), + (None, {"include_usage": True}, None, 1), + ({"include_usage": True}, None, {"include_usage": True}, 0), + (None, None, {"include_usage": True}, 0), + ], + ids=["deployment-config", "router-default", "request-over-deployment", "request-only"], +) +@pytest.mark.asyncio +async def test_router_configured_stream_usage_keeps_a_choice_on_the_usage_chunk( + deployment_stream_options: dict[str, bool] | None, + default_stream_options: dict[str, bool] | None, + request_stream_options: dict[str, bool] | None, + usage_chunk_choice_count: int, +): + deployment_params: Final = {"model": "openai/gpt-5-mini", "api_key": "fake", "mock_response": "hi there"} + router: Final = Router( + model_list=[ + { + "model_name": "m", + "litellm_params": ( + deployment_params + if deployment_stream_options is None + else {**deployment_params, "stream_options": deployment_stream_options} + ), + } + ], + default_litellm_params=None if default_stream_options is None else {"stream_options": default_stream_options}, + ) + request_kwargs: Final = {} if request_stream_options is None else {"stream_options": request_stream_options} + + response: Final = await router.acompletion( + model="m", messages=[{"role": "user", "content": "hi"}], stream=True, **request_kwargs + ) + chunks: Final = [chunk async for chunk in response] + + assert getattr(chunks[-1], "usage", None) is not None + assert len(chunks[-1].choices) == usage_chunk_choice_count + assert all(len(chunk.choices) == 1 for chunk in chunks[:-1]) + + +@pytest.mark.parametrize( + ("stream_options", "marked_injected"), + [({"include_usage": True}, True), (None, False)], + ids=["include-usage", "no-stream-options"], +) +def test_update_kwargs_with_deployment_reads_stream_options_from_pydantic_litellm_params( + stream_options: dict[str, bool] | None, marked_injected: bool +): + from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD + from litellm.types.router import LiteLLM_Params + + kwargs: Final[dict[str, object]] = {"stream": True, "metadata": {}} + Router()._update_kwargs_with_deployment( + deployment={ + "model_name": "gpt-5-mini", + "litellm_params": LiteLLM_Params(model="openai/gpt-5-mini", api_key="fake-key", stream_options=stream_options), + "model_info": {"id": "pydantic-params"}, + }, + kwargs=kwargs, + ) + + assert (kwargs.get(STREAM_USAGE_INJECTED_FIELD) is True) is marked_injected + + @pytest.mark.asyncio async def test_responses_fallback_wrapper_disconnect_billing_keeps_the_primary_streams_tokens(): """After a mid-stream hop, a disconnect bills the primary's relayed text plus the fallback's, not the fallback alone.""" diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 788d405f99f..5a2cefd08b1 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -263,6 +263,7 @@ INTERNAL_STATE_NAMES: Final = ( "litellm_trusted_callback_vars", "_litellm_addressed_response_id", "_litellm_strip_stream_usage", + "_litellm_stream_usage_injected", "client_side_timeout", "model_file_id_mapping", "acompletion", From 8d9b9926d1658df27fd560264f017a1497b81e7a Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 19:32:03 -0700 Subject: [PATCH 042/235] test(perf): add chat_completions_2rps, responses_2rps and messages_2rps direct vs bridge cases (#47752) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/gateway/perf_ci_config.yml | 12 +++++++ tests/e2e/perf/cases/chat_completions_2rps.py | 15 +++++++++ tests/e2e/perf/cases/direct_vs_bridge.py | 31 +++++++++++++++++++ tests/e2e/perf/cases/messages_2rps.py | 16 ++++++++++ tests/e2e/perf/cases/responses_2rps.py | 15 +++++++++ tests/e2e/perf/setup/observe.py | 7 +++++ 6 files changed, 96 insertions(+) create mode 100644 tests/e2e/perf/cases/chat_completions_2rps.py create mode 100644 tests/e2e/perf/cases/direct_vs_bridge.py create mode 100644 tests/e2e/perf/cases/messages_2rps.py create mode 100644 tests/e2e/perf/cases/responses_2rps.py diff --git a/tests/e2e/gateway/perf_ci_config.yml b/tests/e2e/gateway/perf_ci_config.yml index 7ab6637e985..767db313847 100644 --- a/tests/e2e/gateway/perf_ci_config.yml +++ b/tests/e2e/gateway/perf_ci_config.yml @@ -46,6 +46,11 @@ model_list: model: openai/gpt-6-luna api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL api_key: perf-mock-not-checked + - model_name: gpt-6-luna-responses + litellm_params: + model: openai/responses/gpt-6-luna + api_base: os.environ/E2E_PERF_MOCK_OPENAI_URL + api_key: perf-mock-not-checked - model_name: claude-sonnet-5 litellm_params: model: anthropic/claude-sonnet-5 @@ -58,6 +63,13 @@ model_list: aws_region_name: us-east-1 aws_access_key_id: perf-mock-not-checked aws_secret_access_key: perf-mock-not-checked + - model_name: claude-sonnet-5-bedrock-converse + litellm_params: + model: bedrock/converse/anthropic.claude-sonnet-5 + aws_bedrock_runtime_endpoint: os.environ/E2E_PERF_MOCK_BEDROCK_URL + aws_region_name: us-east-1 + aws_access_key_id: perf-mock-not-checked + aws_secret_access_key: perf-mock-not-checked guardrails: - guardrail_name: perf-pre litellm_params: diff --git a/tests/e2e/perf/cases/chat_completions_2rps.py b/tests/e2e/perf/cases/chat_completions_2rps.py new file mode 100644 index 00000000000..9792d99c14f --- /dev/null +++ b/tests/e2e/perf/cases/chat_completions_2rps.py @@ -0,0 +1,15 @@ +"""`/v1/chat/completions` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's chat completions +route, `gpt-6-luna-responses` (`openai/responses/gpt-6-luna`) is bridged to its Responses route.""" + +from __future__ import annotations + +from typing import Final + +from .direct_vs_bridge import direct_vs_bridge + +CASE: Final = direct_vs_bridge( + "chat_completions_2rps", + "chat_completions", + direct="gpt-6-luna", + bridge="gpt-6-luna-responses", +) diff --git a/tests/e2e/perf/cases/direct_vs_bridge.py b/tests/e2e/perf/cases/direct_vs_bridge.py new file mode 100644 index 00000000000..13f609c8299 --- /dev/null +++ b/tests/e2e/perf/cases/direct_vs_bridge.py @@ -0,0 +1,31 @@ +"""The mixed case's traffic on one public API, half of it on a deployment the gateway serves on +that provider's own API (direct) and half on a deployment it has to translate to another provider +API (bridge). Same 2 rps, request count, shapes, stream share, callers, guardrails and limits as +`mixed_size_requests_2rps`, so a gap between the two halves and the mixed baseline is the +translation's cost.""" + +from __future__ import annotations + +from dataclasses import replace + +from setup.performance_test_case import Endpoint, PerformanceTestCase, VirtualKey + +from .mixed_size_requests_2rps import CASE as MIXED + + +def _scoped(key: VirtualKey, direct: str) -> VirtualKey: + return replace(key, models=(direct,)) if key.models else key + + +def direct_vs_bridge(name: str, endpoint: Endpoint, *, direct: str, bridge: str) -> PerformanceTestCase: + return replace( + MIXED, + name=name, + teams=( + replace(MIXED.team("team-a"), models=(direct, bridge)), + replace(MIXED.team("team-b"), models=(bridge,)), + ), + virtual_keys=tuple(_scoped(key, direct) for key in MIXED.virtual_keys), + model_groups=(direct, bridge), + traffic=replace(MIXED.traffic, endpoints={endpoint: 1.0}), + ) diff --git a/tests/e2e/perf/cases/messages_2rps.py b/tests/e2e/perf/cases/messages_2rps.py new file mode 100644 index 00000000000..df5d221d430 --- /dev/null +++ b/tests/e2e/perf/cases/messages_2rps.py @@ -0,0 +1,16 @@ +"""`/v1/messages` at 2 rps: `claude-sonnet-5` is served by the Anthropic mock's Messages route, +`claude-sonnet-5-bedrock-converse` (`bedrock/converse/anthropic.claude-sonnet-5`) is bridged to the +Bedrock mock's Converse route.""" + +from __future__ import annotations + +from typing import Final + +from .direct_vs_bridge import direct_vs_bridge + +CASE: Final = direct_vs_bridge( + "messages_2rps", + "messages", + direct="claude-sonnet-5", + bridge="claude-sonnet-5-bedrock-converse", +) diff --git a/tests/e2e/perf/cases/responses_2rps.py b/tests/e2e/perf/cases/responses_2rps.py new file mode 100644 index 00000000000..dd55d536eb6 --- /dev/null +++ b/tests/e2e/perf/cases/responses_2rps.py @@ -0,0 +1,15 @@ +"""`/v1/responses` at 2 rps: `gpt-6-luna` is served by the OpenAI mock's Responses route, +`claude-sonnet-5` is bridged to the Anthropic mock's Messages route.""" + +from __future__ import annotations + +from typing import Final + +from .direct_vs_bridge import direct_vs_bridge + +CASE: Final = direct_vs_bridge( + "responses_2rps", + "responses", + direct="gpt-6-luna", + bridge="claude-sonnet-5", +) diff --git a/tests/e2e/perf/setup/observe.py b/tests/e2e/perf/setup/observe.py index ccdae6398a9..1532a78606e 100644 --- a/tests/e2e/perf/setup/observe.py +++ b/tests/e2e/perf/setup/observe.py @@ -565,6 +565,7 @@ def _diagnostics(observation: Observation) -> list[str]: return [ " diagnostics (not limits):", f" bodies off reply {observation.bodies_off_reply()}", + f" mock routes {_mock_routes(observation)}", f" guardrail calls per request {len(observation.guardrail_calls) / count:.3f}", f" guardrail missed {observation.guardrail_missed() if observation.guardrail_calls else 'n/a'}", f" retried share {len(observation.retried_markers) / count:.4f}", @@ -575,6 +576,12 @@ def _diagnostics(observation: Observation) -> list[str]: ] +def _mock_routes(observation: Observation) -> str: + """Which provider route each request landed on, so a direct-vs-bridge case shows its split.""" + counts: Final = Counter(f"{seen.provider}/{seen.route}" for seen in observation.mock_requests) + return ", ".join(f"{route} {count}" for route, count in sorted(counts.items())) or "n/a" + + def _datastore_lines(observation: Observation) -> list[str]: """What the per-request Postgres and Redis counts are made of, so a red run names the statement or command that grew. `plumbing` (BEGIN, COMMIT, session settings) and `unclassified` count in neither limit.""" From c8ed5ccca8500ccc0c8a3b499790c07e7e57a763 Mon Sep 17 00:00:00 2001 From: Kazuki Hashimoto <113189196+setouchi-h@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:39:36 +0900 Subject: [PATCH 043/235] fix(router): unregister strategy selectors from global callbacks on discard (#45795) * fix(router): unregister strategy selectors from global callbacks on discard Router.discard only removed its own bound methods from litellm's callback lists, so the strategy selectors it registered (least-busy, usage-based, latency-based, cost-based and routing group selectors) stayed behind and kept receiving events after the router was gone. Reuse the identity based unregister that update_settings already uses so a discarded router takes only its own selectors with it * fix(router): purge discarded selectors from every event callback list function_setup copies litellm.callbacks into the success and failure lists on every request, so a router that served traffic still leaked its selectors there after discard. Route the cleanup through remove_callback_from_all_lists, which now also covers input_callback, and fold the selector gathering into discard so the router coverage check sees no untested helper * fix(router): keep live routers, in-flight hooks and user_config limits intact on discard discard() now rebuilds every global callback list without the router's selectors, optional pre-call checks and own usage callbacks instead of removing them in place, so a callback loop already running never skips a hook. It then re-registers what the remaining live routers need, so a same-strategy survivor keeps its deduped handler. The proxy's per-request user_config router is discarded only after its call returns. * fix(router): bind user_config handlers to the request and tighten discard When a user_config request carries a logging object, its router is discarded before the request runs and its handlers ride on that logging object, so v1 rpm and v2 tpm limits count every request and concurrent requests count once. Without one, the router is discarded after the request as before. The router's pre-call loops also read the request-scoped handlers. discard() now refills only the handler keys it vacated, guards the callback lists and the live-router set with a lock, covers routers built without a model_list, and drops its Slack alerting, adaptive router hook and budget limiter sync task * fix(proxy): reuse one user_config router per config and bind its handlers to each request Every user_config request now goes through a cached router built once per config. Its strategy handlers leave the global callback lists and are bound to the request's logging object instead, so limits count each request once and nothing is left behind after the request. * fix(router): keep the proxy budget limiter on user_config requests A user_config router now swaps in only its strategy selectors, so the proxy's RouterBudgetLimiting stays on the request and keeps counting its spend. Usage logged after a strategy is retired, such as a request still in flight when its user_config router is evicted, now schedules one more push to Redis. * fix(proxy): init the retire fields on the key model budget limiter The key model budget limiter builds its own state instead of calling the base init, so with the proxy's Redis attached every spend increment raised on the missing retire flag and the key's model budget never counted spend. * fix(router): bind the deployment callbacks to user_config requests A cached user_config router serves requests through each request's logging object but bound only its selectors and policies there, so a failing deployment was never cooled down and its usage counters missed requests. * fix(router): retry an evicted router's budget writes and keep the adaptive header on user_config requests A retired budget limiter now keeps retrying its queued Redis increments every sync interval until Redis accepts them, instead of giving up after one failed write. The x-litellm-adaptive-router-model header now comes from one stateless hook in litellm.callbacks that holds no router, so user_config adaptive routers, whose per-router hooks are request-scoped, still answer with the header. * fix(router): lock request callback setup against router cleanup and drain in-flight budget writes on retire function_setup now edits the global callback lists under the same lock that router cleanup uses, so a router dropping its callbacks on another thread can no longer shift the indexes function_setup pops or put a dropped handler back. A budget limiter retired while a Redis write is still in flight now keeps retrying that batch if the write fails, instead of leaving the spend queued with no task left to send it. * fix(utils): set up loggers after releasing the callback lists lock function_setup held the callback lists lock while set_callbacks ran its one-time logger setup, which can block on a Slack package install, so other threads and Router.discard waited on it. The lock now covers only the list reads and writes. * test(router): annotate the retire test's locals as Final * fix(logging): skip replaced global callbacks on the input path too replace_global_callback already kept a same-key global logger out of success and failure dispatch. pre_call and post_call still sent it the input events, so a logger with an input hook could count a request start it never saw finish. Both now share one filter with the success path * test(logging): type the input-path regression test --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 73 ++- .../logging_callback_manager.py | 149 +++-- .../proxy/hooks/model_max_budget_limiter.py | 2 + .../pass_through_endpoints.py | 4 +- litellm/proxy/route_llm_request.py | 59 +- litellm/router.py | 178 ++++-- .../adaptive_router/__init__.py | 7 +- .../router_strategy/adaptive_router/hooks.py | 14 +- .../router_strategy/base_routing_strategy.py | 20 +- litellm/router_strategy/budget_limiter.py | 30 +- litellm/utils.py | 155 ++--- .../router_code_coverage.py | 12 + tests/unit/conftest.py | 4 + .../test_litellm_logging.py | 26 + .../test_logging_callback_manager.py | 2 + tests/unit/proxy/test_route_llm_request.py | 595 +++++++++++++++++- .../adaptive_router/test_hooks.py | 13 +- .../test_base_routing_strategy.py | 28 +- .../test_budget_limiter_hotpath.py | 57 ++ .../test_router/test_router_helper_utils.py | 321 +++++++++- tests/unit/test_utils.py | 108 ++++ 21 files changed, 1656 insertions(+), 201 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 45349bb0e5a..eb2e7b8c240 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -596,6 +596,7 @@ class Logging(LiteLLMLoggingBaseClass): prometheusLogger, \ slack_app custom_pricing: bool = False + replaced_global_callback_keys: frozenset[str] = frozenset() stream_options = None litellm_request_debug: bool = False streamed_anthropic_message_id: str | None = None @@ -650,11 +651,11 @@ class Logging(LiteLLMLoggingBaseClass): litellm_call_id: str, function_id: str, litellm_trace_id: str | None = None, - dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None, + dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None, dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None, dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None, - dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None, - dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None, + dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None, + dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None, applied_guardrails: list[str] | None = None, kwargs: dict | None = None, log_raw_request_response: bool = False, @@ -728,15 +729,17 @@ class Logging(LiteLLMLoggingBaseClass): self.raw_request_only = raw_request_only # Initialize dynamic callbacks - self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks + self.dynamic_input_callbacks: list[str | Callable[..., object] | CustomLogger] | None = dynamic_input_callbacks self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = ( dynamic_success_callbacks ) self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = ( dynamic_async_success_callbacks ) - self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks - self.dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = ( + self.dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = ( + dynamic_failure_callbacks + ) + self.dynamic_async_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = ( dynamic_async_failure_callbacks ) @@ -853,10 +856,32 @@ class Logging(LiteLLMLoggingBaseClass): self.dynamic_async_failure_callbacks, callback ) + def replace_global_callback(self, callback: CustomLogger) -> None: + self.add_dynamic_callback(callback) + self.replaced_global_callback_keys = self.replaced_global_callback_keys | { + litellm.logging_callback_manager.get_custom_logger_key(callback) + } + + def add_dynamic_function_callbacks( + self, + *, + async_success: Callable[..., object], + failure: Callable[..., object], + async_failure: Callable[..., object], + ) -> None: + self.dynamic_async_success_callbacks = self._with_dynamic_callback( + self.dynamic_async_success_callbacks, async_success + ) + self.dynamic_failure_callbacks = self._with_dynamic_callback(self.dynamic_failure_callbacks, failure) + self.dynamic_async_failure_callbacks = self._with_dynamic_callback( + self.dynamic_async_failure_callbacks, async_failure + ) + @staticmethod def _with_dynamic_callback( - callbacks: Sequence[str | Callable | CustomLogger] | None, callback: CustomLogger - ) -> list[str | Callable | CustomLogger]: + callbacks: Sequence[str | Callable[..., object] | CustomLogger] | None, + callback: Callable[..., object] | CustomLogger, + ) -> list[str | Callable[..., object] | CustomLogger]: existing: Final = tuple(callbacks or ()) return [*existing, *(() if callback in existing else (callback,))] @@ -893,9 +918,9 @@ class Logging(LiteLLMLoggingBaseClass): def _process_dynamic_callback_list( self, - callback_list: list[str | Callable | CustomLogger] | None, + callback_list: list[str | Callable[..., object] | CustomLogger] | None, dynamic_callbacks_type: Literal["input", "success", "failure", "async_success", "async_failure"], - ) -> list[str | Callable | CustomLogger] | None: + ) -> list[str | Callable[..., object] | CustomLogger] | None: """ Helper function to initialize CustomLogger compatible callbacks in self.dynamic_* callbacks @@ -907,7 +932,7 @@ class Logging(LiteLLMLoggingBaseClass): if callback_list is None: return None - processed_list: Final[list[str | Callable | CustomLogger]] = [] + processed_list: Final[list[str | Callable[..., object] | CustomLogger]] = [] for callback in callback_list: if isinstance(callback, str) and callback in litellm._known_custom_logger_compatible_callbacks: for callback_instance in self._resolve_dynamic_callback_string(callback): @@ -1568,7 +1593,10 @@ class Logging(LiteLLMLoggingBaseClass): "api_call_start_time" ] # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made - callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or []) + callbacks: Final = ( + *self._without_replaced_global_callbacks(litellm.input_callback), + *(self.dynamic_input_callbacks or ()), + ) for callback in callbacks: try: if callback == "supabase" and supabaseClient is not None: @@ -1741,7 +1769,10 @@ class Logging(LiteLLMLoggingBaseClass): ) # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made - callbacks: Final = litellm.input_callback + (self.dynamic_input_callbacks or []) + callbacks: Final = ( + *self._without_replaced_global_callbacks(litellm.input_callback), + *(self.dynamic_input_callbacks or ()), + ) for callback in callbacks: try: if callback == "sentry" and add_breadcrumb: @@ -4518,14 +4549,26 @@ class Logging(LiteLLMLoggingBaseClass): _filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks) return len(_filtered_failure_callbacks) > 0 + def _without_replaced_global_callbacks( + self, global_callbacks: Iterable[str | Callable[..., object] | CustomLogger] + ) -> tuple[str | Callable[..., object] | CustomLogger, ...]: + if not self.replaced_global_callback_keys: + return tuple(global_callbacks) + return tuple( + litellm.logging_callback_manager.without_custom_logger_keys( + global_callbacks, self.replaced_global_callback_keys + ) + ) + def get_combined_callback_list( self, dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None, global_callbacks: list[str | Callable[..., object] | CustomLogger], ) -> list[str | Callable[..., object] | CustomLogger]: + kept_global_callbacks: Final = self._without_replaced_global_callbacks(global_callbacks) if dynamic_success_callbacks is None: - return list(global_callbacks) - return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks)) + return list(kept_global_callbacks) + return list(dict.fromkeys((*dynamic_success_callbacks, *kept_global_callbacks))) def _remove_internal_litellm_callbacks(self, callbacks: list) -> list: """ diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index 9f521ffc6cc..307353d081b 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -1,4 +1,6 @@ -from collections.abc import Callable +import threading +from collections.abc import Callable, Iterable, Iterator +from types import MethodType from typing import TYPE_CHECKING, Final import litellm @@ -15,6 +17,7 @@ else: _custom_logger_compatible_callbacks_literal = str _generic_api_logger_cache: Final[dict[str, GenericAPILogger]] = {} +_callback_lists_lock: Final = threading.RLock() class LoggingCallbackManager: @@ -43,16 +46,18 @@ class LoggingCallbackManager: Add a input callback to litellm.input_callback. Auto-routes async callbacks to litellm._async_input_callback. """ - if not isinstance(callback, str) and self._is_async_callable(callback): - self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback) - else: - self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback) + with _callback_lists_lock: + if not isinstance(callback, str) and self._is_async_callable(callback): + self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_input_callback) + else: + self._safe_add_callback_to_list(callback=callback, parent_list=litellm.input_callback) def add_litellm_service_callback(self, callback: CustomLogger | str | Callable[..., object]): """ Add a service callback to litellm.service_callback """ - self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback) + with _callback_lists_lock: + self._safe_add_callback_to_list(callback=callback, parent_list=litellm.service_callback) def add_litellm_callback(self, callback: CustomLogger | str | Callable[..., object]): """ @@ -60,10 +65,11 @@ class LoggingCallbackManager: Ensures no duplicates are added. """ - self._safe_add_callback_to_list( - callback=callback, - parent_list=litellm.callbacks, - ) + with _callback_lists_lock: + self._safe_add_callback_to_list( + callback=callback, + parent_list=litellm.callbacks, + ) def add_litellm_success_callback(self, callback: CustomLogger | str | Callable[..., object]): """ @@ -71,37 +77,41 @@ class LoggingCallbackManager: Auto-routes async callbacks to litellm._async_success_callback. Special-cases 'dynamodb' and 'openmeter' as async callbacks. """ - if ( - isinstance(callback, str) - and callback in ("dynamodb", "openmeter") - or not isinstance(callback, str) - and self._is_async_callable(callback) - ): - self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback) - else: - self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback) + with _callback_lists_lock: + if ( + isinstance(callback, str) + and callback in ("dynamodb", "openmeter") + or not isinstance(callback, str) + and self._is_async_callable(callback) + ): + self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback) + else: + self._safe_add_callback_to_list(callback=callback, parent_list=litellm.success_callback) def add_litellm_failure_callback(self, callback: CustomLogger | str | Callable[..., object]): """ Add a failure callback to `litellm.failure_callback`. Auto-routes async callbacks to litellm._async_failure_callback. """ - if not isinstance(callback, str) and self._is_async_callable(callback): - self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback) - else: - self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback) + with _callback_lists_lock: + if not isinstance(callback, str) and self._is_async_callable(callback): + self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback) + else: + self._safe_add_callback_to_list(callback=callback, parent_list=litellm.failure_callback) def add_litellm_async_success_callback(self, callback: CustomLogger | Callable[..., object] | str): """ Add a success callback to litellm._async_success_callback """ - self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback) + with _callback_lists_lock: + self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_success_callback) def add_litellm_async_failure_callback(self, callback: CustomLogger | Callable[..., object] | str): """ Add a failure callback to litellm._async_failure_callback """ - self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback) + with _callback_lists_lock: + self._safe_add_callback_to_list(callback=callback, parent_list=litellm._async_failure_callback) def remove_callback_from_list_by_object(self, callback_list, obj, require_self=True): """ @@ -110,13 +120,14 @@ class LoggingCallbackManager: if not isinstance(callback_list, list): # Not list -> do nothing return - if require_self: - remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj] - else: - remove_list = [c for c in callback_list if c == obj] + with _callback_lists_lock: + if require_self: + remove_list = [c for c in callback_list if hasattr(c, "__self__") and c.__self__ == obj] + else: + remove_list = [c for c in callback_list if c == obj] - for c in remove_list: - callback_list.remove(c) + for c in remove_list: + callback_list.remove(c) def remove_callbacks_by_type(self, callback_list, callback_type): """ @@ -134,10 +145,11 @@ class LoggingCallbackManager: if not isinstance(callback_list, list): return - remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)] + with _callback_lists_lock: + remove_list: Final = [c for c in callback_list if isinstance(c, callback_type)] - for c in remove_list: - callback_list.remove(c) + for c in remove_list: + callback_list.remove(c) def _add_string_callback_to_list( self, callback: str, parent_list: list[CustomLogger | Callable[..., object] | str] @@ -336,6 +348,55 @@ class LoggingCallbackManager: return "-".join(key_parts) + get_custom_logger_key = _get_custom_logger_key + callback_lists_lock: Final = _callback_lists_lock + + def drop_callbacks(self, callbacks: Iterable[object], method_owners: Iterable[object] = ()) -> frozenset[str]: + dropped_ids: Final = frozenset(map(id, callbacks)) + owner_ids: Final = frozenset(map(id, method_owners)) + + def keep(callback: object) -> bool: + if id(callback) in dropped_ids: + return False + return not (isinstance(callback, MethodType) and id(callback.__self__) in owner_ids) + + with _callback_lists_lock: + vacated_keys: Final = frozenset( + self._get_custom_logger_key(callback) + for callback in litellm.callbacks + if isinstance(callback, CustomLogger) and id(callback) in dropped_ids + ) + litellm.callbacks = [callback for callback in litellm.callbacks if keep(callback)] + litellm.input_callback = [callback for callback in litellm.input_callback if keep(callback)] + litellm.success_callback = [callback for callback in litellm.success_callback if keep(callback)] + litellm.failure_callback = [callback for callback in litellm.failure_callback if keep(callback)] + litellm.service_callback = [callback for callback in litellm.service_callback if keep(callback)] + litellm._async_success_callback = [ + callback for callback in litellm._async_success_callback if keep(callback) + ] + litellm._async_failure_callback = [ + callback for callback in litellm._async_failure_callback if keep(callback) + ] + return vacated_keys + + def with_custom_logger_keys( + self, candidates: Iterable[CustomLogger], keys: frozenset[str] + ) -> tuple[CustomLogger, ...]: + if not keys: + return () + class_names: Final = frozenset(key.split("-", 1)[0] for key in keys) + same_class: Final = (candidate for candidate in candidates if type(candidate).__name__ in class_names) + return tuple(candidate for candidate in same_class if self._get_custom_logger_key(candidate) in keys) + + def without_custom_logger_keys( + self, callbacks: Iterable[str | Callable[..., object] | CustomLogger], keys: frozenset[str] + ) -> Iterator[str | Callable[..., object] | CustomLogger]: + return ( + callback + for callback in callbacks + if not (isinstance(callback, CustomLogger) and self._get_custom_logger_key(callback) in keys) + ) + def _reset_all_callbacks(self): """ Reset all callbacks to an empty list @@ -363,19 +424,21 @@ class LoggingCallbackManager: _get_all_callbacks = get_all_callbacks - def remove_callback_from_all_lists(self, obj, require_self=False) -> None: + def remove_callback_from_all_lists(self, obj: object, require_self: bool = False) -> None: """ Remove a callback object from every callback list it may have been promoted into, so a re-initialized callback leaves no stale instance behind. """ - for callback_list in ( - litellm.callbacks, - litellm.success_callback, - litellm.failure_callback, - litellm._async_success_callback, - litellm._async_failure_callback, - ): - self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self) + with _callback_lists_lock: + for callback_list in ( + litellm.callbacks, + litellm.input_callback, + litellm.success_callback, + litellm.failure_callback, + litellm._async_success_callback, + litellm._async_failure_callback, + ): + self.remove_callback_from_list_by_object(callback_list, obj, require_self=require_self) def get_active_additional_logging_utils_from_custom_logger( self, diff --git a/litellm/proxy/hooks/model_max_budget_limiter.py b/litellm/proxy/hooks/model_max_budget_limiter.py index 37cef41034b..bc9cee1cb1d 100644 --- a/litellm/proxy/hooks/model_max_budget_limiter.py +++ b/litellm/proxy/hooks/model_max_budget_limiter.py @@ -374,6 +374,8 @@ class _PROXY_VirtualKeyModelMaxBudgetLimiter(RouterBudgetLimiting): self._redis_increment_queue_lock = asyncio.Lock() self._redis_increment_flush_lock = asyncio.Lock() self._detached_increment_operations = None + self._retired = False + self._flush_after_retire = None self.deployment_budget_config = None @with_service_target("model_budgets") diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index fe7ac30e434..e1e8864678e 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -880,8 +880,8 @@ def _build_passthrough_failure_request_payload( @dataclass(frozen=True, slots=True) class _TeamCallbackWiring: - success_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg - failure_callbacks: "list[str | Callable | CustomLogger] | None" = None # mutable-ok: Logging.__init__ arg + success_callbacks: "list[str | Callable[..., object] | CustomLogger] | None" = None # mutable-ok: Logging init arg + failure_callbacks: "list[str | Callable[..., object] | CustomLogger] | None" = None # mutable-ok: Logging init arg logging_kwargs: dict[str, str | dict[str, str]] | None = None # mutable-ok: Logging.__init__ arg diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 25f9f1dff15..cb226929167 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -1,8 +1,13 @@ import asyncio -from collections.abc import Mapping +import hashlib +import json +from collections import OrderedDict +from collections.abc import Callable, Mapping from dataclasses import dataclass +from datetime import datetime from types import MappingProxyType from typing import TYPE_CHECKING, Annotated, Any, Final, Literal +from uuid import uuid4 import httpx from fastapi import HTTPException, status @@ -10,6 +15,7 @@ from pydantic import BeforeValidator, TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._types import ProxyException, UserAPIKeyAuth from litellm.router_utils.common_utils import is_proxy_admin_request @@ -54,7 +60,34 @@ _ROUTER_ONLY_SETTINGS: Final = frozenset( ) -def _route_user_config_request(data: dict, route_type: str): +class UserConfigRouters: + def __init__(self, max_size: int) -> None: + self._max_size: Final = max_size + self._routers: Final[OrderedDict[str, LitellmRouter]] = OrderedDict() # mutable-ok: LRU evicts in place + + def get(self, config: Mapping[str, object], build: Callable[[], LitellmRouter]) -> LitellmRouter: + key: Final = hashlib.sha256(json.dumps(config, sort_keys=True, default=str).encode()).hexdigest() + cached: Final = self._routers.get(key) + if cached is not None: + self._routers.move_to_end(key) + return cached + router: Final = build() + router.serve_through_request_logging() + self._routers[key] = router + if len(self._routers) > self._max_size: + self._routers.popitem(last=False)[1].discard() + return router + + def clear(self) -> None: + for router in self._routers.values(): + router.discard() + self._routers.clear() + + +user_config_routers: Final = UserConfigRouters(max_size=32) + + +async def _route_user_config_request(data: dict, route_type: str): """Route a request using the user-provided router config.""" router_config: Final = data.pop("user_config") @@ -63,10 +96,24 @@ def _route_user_config_request(data: dict, route_type: str): valid_args: Final = litellm.Router.get_valid_args() filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args} - user_router: Final = litellm.Router(**filtered_config) - ret_val: Final = getattr(user_router, f"{route_type}")(**data) - user_router.discard() - return ret_val + logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type) + cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config)) + cached_router.bind_to_request_logging(logging_obj) + return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj}) + + +def _request_logging_obj(data: Mapping[str, object], route_type: str) -> LiteLLMLoggingObj: + arrived_with: Final = data.get("litellm_logging_obj") + if isinstance(arrived_with, LiteLLMLoggingObj): + return arrived_with + logging_obj, _ = litellm.utils.function_setup( + original_function=route_type, + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), # noqa: DTZ005 # Logging subtracts it from naive end times + is_async_call=True, + **{"litellm_call_id": str(uuid4()), **data}, + ) + return logging_obj def _is_a2a_agent_model(model_name: object) -> bool: diff --git a/litellm/router.py b/litellm/router.py index 1ce1e0a5a70..0ac21960be1 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -38,6 +38,7 @@ from contextvars import ContextVar from dataclasses import dataclass from datetime import datetime, timezone from functools import lru_cache, partial +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, TypeAlias, TypeVar, Union, cast @@ -553,6 +554,7 @@ _PreRoutingStrategyT = TypeVar("_PreRoutingStrategyT") _CallbackT = TypeVar("_CallbackT") _DeploymentT = TypeVar("_DeploymentT", bound=Mapping[str, object]) + _ALIAS_PARAMS_NEVER_FORWARDED: Final = frozenset( {"model", "api_base", "api_key", "api_version", "responses_api_version"} ) @@ -1047,6 +1049,8 @@ class RoutingArgs(enum.Enum): # as the per-request one built from a caller-supplied user_config, drops out on its # own rather than leaving entries behind that nothing can withdraw. _live_routers: Final["weakref.WeakSet[Router]"] = weakref.WeakSet() +_routers_holding_global_callbacks: Final["weakref.WeakSet[Router]"] = weakref.WeakSet() +_routers_holding_global_callbacks_lock: Final = threading.Lock() def _replay_live_router_model_cost() -> None: @@ -1579,22 +1583,18 @@ class Router: self._init_routing_groups(self._routing_groups_input) self._override_selectors: dict[str, RouterStrategySelector | None] = {} self._override_selectors_lock = threading.Lock() + self._request_scoped_selectors: tuple[CustomLogger, ...] = () + self._request_scoped_policies: tuple[CustomLogger, ...] = () self.access_groups = None + with _routers_holding_global_callbacks_lock: + _routers_holding_global_callbacks.add(self) ## USAGE TRACKING ## - if isinstance(litellm._async_success_callback, list): - litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success) - else: - litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success) + litellm.logging_callback_manager.add_litellm_async_success_callback(self.deployment_callback_on_success) if isinstance(litellm.success_callback, list): litellm.logging_callback_manager.add_litellm_success_callback(self.sync_deployment_callback_on_success) else: litellm.success_callback = [self.sync_deployment_callback_on_success] - if isinstance(litellm._async_failure_callback, list): - litellm.logging_callback_manager.add_litellm_async_failure_callback( - self.async_deployment_callback_on_failure - ) - else: - litellm._async_failure_callback = [self.async_deployment_callback_on_failure] + litellm.logging_callback_manager.add_litellm_async_failure_callback(self.async_deployment_callback_on_failure) ## COOLDOWNS ## if isinstance(litellm.failure_callback, list): litellm.logging_callback_manager.add_litellm_failure_callback(self.deployment_callback_on_failure) @@ -1690,20 +1690,116 @@ class Router: # Stop contributing to cost-map rebuilds straight away rather than waiting # for this router to be collected. _live_routers.discard(self) - litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm._async_success_callback, self) - litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.success_callback, self) - litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm._async_failure_callback, self) - litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.failure_callback, self) - litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.input_callback, self) - litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.service_callback, self) - litellm.logging_callback_manager.remove_callback_from_list_by_object(litellm.callbacks, self) + self.retire_routing_state() + self._release_global_callbacks() - # Remove ForwardClientSideHeadersByModelGroup if it exists - if self.optional_callbacks is not None: - for callback in self.optional_callbacks: - litellm.logging_callback_manager.remove_callback_from_list_by_object( - litellm.callbacks, callback, require_self=False - ) + def _release_global_callbacks(self) -> None: + with _routers_holding_global_callbacks_lock: + _routers_holding_global_callbacks.discard(self) + slack_alerting_loggers: Final = self._slack_alerting_loggers() + adaptive_router_hooks: Final = self._adaptive_router_hooks() + vacated_keys: Final = litellm.logging_callback_manager.drop_callbacks( + ( + *self._callback_selectors(), + *self._override_selectors.values(), + *(self.optional_callbacks or ()), + *slack_alerting_loggers, + *adaptive_router_hooks, + ), + method_owners=(self, *slack_alerting_loggers), + ) + with _routers_holding_global_callbacks_lock: + survivors: Final = tuple(_routers_holding_global_callbacks) + candidates: Final = chain.from_iterable(router._global_callback_candidates() for router in survivors) + for candidate in litellm.logging_callback_manager.with_custom_logger_keys(candidates, vacated_keys): + self._register_router_selector(candidate) + adaptive_keys: Final = frozenset( + map(litellm.logging_callback_manager.get_custom_logger_key, adaptive_router_hooks) + ) + adaptive_survivor: Final = next((router for router in survivors if router.adaptive_routers), None) + if adaptive_survivor is not None and vacated_keys & adaptive_keys: + adaptive_survivor._sync_adaptive_router_hooks() + + def serve_through_request_logging(self) -> None: + self._request_scoped_selectors = (*self._callback_selectors(), *self._request_adaptive_router_hooks()) + self._request_scoped_policies = (*self._optional_loggers(), *self._slack_alerting_loggers()) + self._release_global_callbacks() + + def bind_to_request_logging(self, logging_obj: LiteLLMLogging) -> None: + logging_obj.add_dynamic_function_callbacks( + async_success=self.deployment_callback_on_success, + failure=self.deployment_callback_on_failure, + async_failure=self.async_deployment_callback_on_failure, + ) + for selector in self._request_scoped_selectors: + logging_obj.replace_global_callback(selector) + for policy in self._request_policies_not_set_globally(): + logging_obj.add_dynamic_callback(policy) + + def _request_policies_not_set_globally(self) -> tuple[CustomLogger, ...]: + global_keys: Final = frozenset( + map( + litellm.logging_callback_manager.get_custom_logger_key, + (callback for callback in litellm.callbacks if isinstance(callback, CustomLogger)), + ) + ) + return tuple( + policy + for policy in self._request_scoped_policies + if litellm.logging_callback_manager.get_custom_logger_key(policy) not in global_keys + ) + + def _request_adaptive_router_hooks(self) -> tuple[CustomLogger, ...]: + from litellm.router_strategy.adaptive_router.hooks import ( + AdaptiveRouterPostCallHook, + ) + + return tuple( + AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy) + for tagged in chain.from_iterable(self.adaptive_routers.values()) + ) + + def _routing_callbacks(self) -> Iterable[object]: + if not (self._request_scoped_selectors or self._request_scoped_policies): + return litellm.callbacks + replaced_keys: Final = frozenset( + map(litellm.logging_callback_manager.get_custom_logger_key, self._request_scoped_selectors) + ) + return chain( + litellm.logging_callback_manager.without_custom_logger_keys(litellm.callbacks, replaced_keys), + self._request_scoped_selectors, + self._request_policies_not_set_globally(), + ) + + def retire_routing_state(self) -> None: + for strategy in (*self._global_callback_candidates(), *self._override_selectors.values()): + if isinstance(strategy, BaseRoutingStrategy | RouterBudgetLimiting): + strategy.retire() + + def _slack_alerting_loggers(self) -> tuple[CustomLogger, ...]: + slack_alerting_logger: Final = getattr(self, "slack_alerting_logger", None) + return (slack_alerting_logger,) if isinstance(slack_alerting_logger, CustomLogger) else () + + def _adaptive_router_hooks(self) -> tuple[CustomLogger, ...]: + from litellm.router_strategy.adaptive_router.hooks import ( + AdaptiveRouterPostCallHook, + ) + + own_strategy_ids: Final = frozenset( + id(tagged.strategy) for tagged in chain.from_iterable(self.adaptive_routers.values()) + ) + hooks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook) + return tuple( + hook + for hook in hooks + if isinstance(hook, AdaptiveRouterPostCallHook) and id(hook.adaptive_router) in own_strategy_ids + ) + + def _optional_loggers(self) -> tuple[CustomLogger, ...]: + return tuple(callback for callback in self.optional_callbacks or () if isinstance(callback, CustomLogger)) + + def _global_callback_candidates(self) -> tuple[CustomLogger, ...]: + return (*self._callback_selectors(), *self._optional_loggers(), *self._slack_alerting_loggers()) @staticmethod def _create_redis_cache( @@ -1809,7 +1905,7 @@ class Router: return selector @staticmethod - def _register_router_selector(selector: RouterStrategySelector) -> None: + def _register_router_selector(selector: CustomLogger) -> None: if isinstance(selector, LeastBusyLoggingHandler): if isinstance(litellm.input_callback, list): litellm.logging_callback_manager.add_litellm_input_callback(selector) @@ -1818,23 +1914,28 @@ class Router: if isinstance(litellm.callbacks, list): litellm.logging_callback_manager.add_litellm_callback(selector) + def _callback_selectors(self) -> tuple[RouterStrategySelector, ...]: + return tuple( + selector + for selector in ( + *(getattr(self, attr, None) for attr in self._DEFAULT_SELECTOR_ATTR_BY_STRATEGY.values()), + *chain.from_iterable(selectors.values() for selectors in self._group_selectors.values()), + ) + if selector is not None + ) + def _unregister_router_selectors(self, selectors: Sequence[object]) -> None: """ Drop router-owned strategy selectors from litellm's global callback - lists by identity. Used before re-init (`routing_strategy_init` / - `_init_routing_groups`) so repeated `update_settings` calls don't - accumulate dead selectors that keep receiving callback events. + lists. Used before re-init (`routing_strategy_init` / + `_init_routing_groups`) and by `discard` so dead selectors stop + receiving callback events. `function_setup` copies `litellm.callbacks` + into the per-event lists on every request, so those are purged too. """ for selector in selectors: if isinstance(selector, BaseRoutingStrategy): selector.retire() - selector_ids: Final = {id(s) for s in selectors if s is not None} - if not selector_ids: - return - if isinstance(litellm.callbacks, list): - litellm.callbacks = [c for c in litellm.callbacks if id(c) not in selector_ids] - if isinstance(litellm.input_callback, list): - litellm.input_callback = [c for c in litellm.input_callback if id(c) not in selector_ids] + litellm.logging_callback_manager.drop_callbacks(selectors) def _apply_updated_routing_strategy_args(self) -> None: """ @@ -9804,7 +9905,7 @@ class Router: Raises: - Rate Limit Exception - If the deployment is over it's tpm/rpm limits """ - for _callback in litellm.callbacks: + for _callback in self._routing_callbacks(): if isinstance(_callback, CustomLogger): _callback.pre_call_check(deployment) @@ -9825,7 +9926,7 @@ class Router: Raises: - Rate Limit Exception - If the deployment is over it's tpm/rpm limits """ - for _callback in litellm.callbacks: + for _callback in self._routing_callbacks(): if isinstance(_callback, CustomLogger): try: await _callback.async_pre_call_check(deployment, parent_otel_span) @@ -9962,7 +10063,7 @@ class Router: - Rate Limit Exception - If the deployment is over it's tpm/rpm limits """ returned_healthy_deployments = healthy_deployments - for _callback in litellm.callbacks: + for _callback in self._routing_callbacks(): if isinstance(_callback, CustomLogger): try: returned_healthy_deployments = await _callback.async_filter_deployments( @@ -10617,6 +10718,7 @@ class Router: changes, otherwise a released router keeps recording turns through its hook.""" from litellm.router_strategy.adaptive_router.hooks import ( AdaptiveRouterPostCallHook, + AdaptiveRouterResponseHeaderHook, ) for callback in litellm.logging_callback_manager.get_custom_loggers_for_type(AdaptiveRouterPostCallHook): @@ -10626,6 +10728,8 @@ class Router: litellm.logging_callback_manager.add_litellm_callback( AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy) ) + if self.adaptive_routers: + litellm.logging_callback_manager.add_litellm_callback(AdaptiveRouterResponseHeaderHook()) def init_adaptive_router_deployment(self, deployment: Deployment) -> None: """ diff --git a/litellm/router_strategy/adaptive_router/__init__.py b/litellm/router_strategy/adaptive_router/__init__.py index d7f55ebced9..1f7e6b93169 100644 --- a/litellm/router_strategy/adaptive_router/__init__.py +++ b/litellm/router_strategy/adaptive_router/__init__.py @@ -1,6 +1,9 @@ """Adaptive router strategy. See README.md for design overview.""" from litellm.router_strategy.adaptive_router.adaptive_router import AdaptiveRouter -from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook +from litellm.router_strategy.adaptive_router.hooks import ( + AdaptiveRouterPostCallHook, + AdaptiveRouterResponseHeaderHook, +) -__all__ = ["AdaptiveRouter", "AdaptiveRouterPostCallHook"] +__all__ = ["AdaptiveRouter", "AdaptiveRouterPostCallHook", "AdaptiveRouterResponseHeaderHook"] diff --git a/litellm/router_strategy/adaptive_router/hooks.py b/litellm/router_strategy/adaptive_router/hooks.py index 92207e07c52..3427b81be68 100644 --- a/litellm/router_strategy/adaptive_router/hooks.py +++ b/litellm/router_strategy/adaptive_router/hooks.py @@ -170,12 +170,7 @@ def _assistant_content_and_tool_calls(response_obj: Any) -> tuple[object, Sequen return content, tool_calls -class AdaptiveRouterPostCallHook(CustomLogger): - """One hook instance per AdaptiveRouter. Registered into litellm.callbacks.""" - - def __init__(self, adaptive_router: AdaptiveRouter) -> None: - self.adaptive_router = adaptive_router - +class AdaptiveRouterResponseHeaderHook(CustomLogger): async def async_post_call_response_headers_hook( self, data: Mapping[str, object], @@ -200,6 +195,13 @@ class AdaptiveRouterPostCallHook(CustomLogger): return None return {ADAPTIVE_ROUTER_RESPONSE_HEADER: chosen} + +class AdaptiveRouterPostCallHook(CustomLogger): + """One hook instance per AdaptiveRouter. Registered into litellm.callbacks.""" + + def __init__(self, adaptive_router: AdaptiveRouter) -> None: + self.adaptive_router = adaptive_router + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): await self._record(kwargs, response_obj, response_status=200) diff --git a/litellm/router_strategy/base_routing_strategy.py b/litellm/router_strategy/base_routing_strategy.py index 51400e6da24..2cea7389808 100644 --- a/litellm/router_strategy/base_routing_strategy.py +++ b/litellm/router_strategy/base_routing_strategy.py @@ -24,6 +24,8 @@ class BaseRoutingStrategy(ABC): self.dual_cache = dual_cache self.redis_increment_operation_queue: list[RedisPipelineIncrementOperation] = [] self._sync_task: asyncio.Task[None] | None = None + self._retired = False + self._flush_after_retire: asyncio.Task[None] | None = None if should_batch_redis_writes: self.setup_sync_task(default_sync_interval) @@ -47,13 +49,25 @@ class BaseRoutingStrategy(ABC): def retire(self) -> None: self.cancel_sync_task() - if not self.redis_increment_operation_queue: + self._retired = True + self._schedule_flush_after_retire() + + def _schedule_flush_after_retire(self) -> None: + if self._flush_after_retire is not None and not self._flush_after_retire.done(): + return + if not self.redis_increment_operation_queue or self.dual_cache.redis_cache is None: return try: loop: Final = asyncio.get_running_loop() except RuntimeError: return - loop.create_task(self._push_in_memory_increments_to_redis()) + self._flush_after_retire = loop.create_task(self._drain_increments_after_retire()) + + async def _drain_increments_after_retire(self) -> None: + if not self.redis_increment_operation_queue: + return + await self._push_in_memory_increments_to_redis() + await self._drain_increments_after_retire() async def cleanup(self): """Cleanup method to be called when shutting down""" @@ -98,6 +112,8 @@ class BaseRoutingStrategy(ABC): self.redis_increment_operation_queue.append(increment_op) self.add_to_in_memory_keys_to_update(key=key) + if self._retired: + self._schedule_flush_after_retire() return result @with_service_target("router_usage") diff --git a/litellm/router_strategy/budget_limiter.py b/litellm/router_strategy/budget_limiter.py index 147ce60a99b..eab232eae18 100644 --- a/litellm/router_strategy/budget_limiter.py +++ b/litellm/router_strategy/budget_limiter.py @@ -128,7 +128,9 @@ class RouterBudgetLimiting(CustomLogger): self._redis_increment_queue_lock = asyncio.Lock() self._redis_increment_flush_lock = asyncio.Lock() self._detached_increment_operations: tuple[RedisPipelineIncrementOperation, ...] | None = None - asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis()) + self._sync_task: asyncio.Task[None] = asyncio.create_task(self.periodic_sync_in_memory_spend_with_redis()) + self._retired = False + self._flush_after_retire: asyncio.Task[None] | None = None self.provider_budget_config: GenericBudgetConfigType | None = provider_budget_config self.deployment_budget_config: GenericBudgetConfigType | None = None self.tag_budget_config: GenericBudgetConfigType | None = None @@ -140,6 +142,30 @@ class RouterBudgetLimiting(CustomLogger): if isinstance(litellm.callbacks, list): litellm.logging_callback_manager.add_litellm_callback(self) + def retire(self) -> None: + self._sync_task.cancel() + self._retired = True + self._schedule_flush_after_retire() + + def _schedule_flush_after_retire(self) -> None: + if self._flush_after_retire is not None and not self._flush_after_retire.done(): + return + if not self._has_unwritten_increments(): + return + try: + loop: Final = asyncio.get_running_loop() + except RuntimeError: + return + self._flush_after_retire = loop.create_task(self._drain_increments_after_retire()) + + def _has_unwritten_increments(self) -> bool: + return bool(self.redis_increment_operation_queue) or self._detached_increment_operations is not None + + async def _drain_increments_after_retire(self) -> None: + while self._has_unwritten_increments(): + if not await self._push_in_memory_increments_to_redis(): + await asyncio.sleep(DEFAULT_REDIS_SYNC_INTERVAL) + @with_service_target("router_budgets") async def async_filter_deployments( self, @@ -427,6 +453,8 @@ class RouterBudgetLimiting(CustomLogger): if self.dual_cache.redis_cache is None: return self.redis_increment_operation_queue.append(increment_op) + if self._retired: + self._schedule_flush_after_retire() def _get_redis_increment_queue_lock(self) -> asyncio.Lock: return self._redis_increment_queue_lock diff --git a/litellm/utils.py b/litellm/utils.py index 1aad9baf06a..1a114ffe9f0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1126,83 +1126,90 @@ def function_setup( dynamic_callbacks: Final[list[str | Callable | CustomLogger] | None] = kwargs.pop("callbacks", None) all_callbacks: Final = get_dynamic_callbacks(dynamic_callbacks=dynamic_callbacks) - if len(all_callbacks) > 0: - for callback in all_callbacks: - # check if callback is a string - e.g. "lago", "openmeter" - if isinstance(callback, str): - callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class( - callback, - internal_usage_cache=None, - llm_router=None, - ) - if callback is None or any( - type(cb) is type(callback) for cb in litellm._async_success_callback - ): # don't double add a callback - continue - if callback not in litellm.input_callback: - litellm.input_callback.append(callback) - if callback not in litellm.success_callback: - litellm.logging_callback_manager.add_litellm_success_callback(callback) - if callback not in litellm.failure_callback: - litellm.logging_callback_manager.add_litellm_failure_callback(callback) - if callback not in litellm._async_success_callback: - litellm.logging_callback_manager.add_litellm_async_success_callback(callback) - if callback not in litellm._async_failure_callback: - litellm.logging_callback_manager.add_litellm_async_failure_callback(callback) - print_verbose(f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}") + with litellm.logging_callback_manager.callback_lists_lock: + if len(all_callbacks) > 0: + for callback in all_callbacks: + # check if callback is a string - e.g. "lago", "openmeter" + if isinstance(callback, str): + callback = litellm.litellm_core_utils.litellm_logging._init_custom_logger_compatible_class( + callback, + internal_usage_cache=None, + llm_router=None, + ) + if callback is None or any( + type(cb) is type(callback) for cb in litellm._async_success_callback + ): # don't double add a callback + continue + if callback not in litellm.input_callback: + litellm.input_callback.append(callback) + if callback not in litellm.success_callback: + litellm.logging_callback_manager.add_litellm_success_callback(callback) + if callback not in litellm.failure_callback: + litellm.logging_callback_manager.add_litellm_failure_callback(callback) + if callback not in litellm._async_success_callback: + litellm.logging_callback_manager.add_litellm_async_success_callback(callback) + if callback not in litellm._async_failure_callback: + litellm.logging_callback_manager.add_litellm_async_failure_callback(callback) + print_verbose( + f"Initialized litellm callbacks, Async Success Callbacks: {litellm._async_success_callback}" + ) - if ( - len(litellm.input_callback) > 0 or len(litellm.success_callback) > 0 or len(litellm.failure_callback) > 0 - ) and len(callback_list) == 0: - callback_list = list(set(litellm.input_callback + litellm.success_callback + litellm.failure_callback)) + callbacks_to_set_up: Final = ( + list(set(litellm.input_callback + litellm.success_callback + litellm.failure_callback)) + if len(callback_list) == 0 + else [] + ) + if callbacks_to_set_up: + callback_list = callbacks_to_set_up + ## ASYNC CALLBACKS - safety net for callbacks added via direct append + if len(litellm.input_callback) > 0: + removed_async_items = [] + for index, callback in enumerate(litellm.input_callback): + if coroutine_checker.is_async_callable(callback): + litellm._async_input_callback.append(callback) + removed_async_items.append(index) + + # Pop the async items from input_callback in reverse order to avoid index issues + for index in reversed(removed_async_items): + litellm.input_callback.pop(index) + if len(litellm.success_callback) > 0: + removed_async_items = [] + for index, callback in enumerate(litellm.success_callback): + if coroutine_checker.is_async_callable(callback): + litellm.logging_callback_manager.add_litellm_async_success_callback(callback) + removed_async_items.append(index) + elif callback == "dynamodb" or callback == "openmeter": + # dynamo is an async callback, it's used for the proxy and needs to be async + # we only support async dynamo db logging for acompletion/aembedding since that's used on proxy + litellm.logging_callback_manager.add_litellm_async_success_callback(callback) + removed_async_items.append(index) + elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str): + add_custom_logger_callback_to_specific_event(callback, "success") + + # Pop the async items from success_callback in reverse order to avoid index issues + for index in reversed(removed_async_items): + litellm.success_callback.pop(index) + + if len(litellm.failure_callback) > 0: + removed_async_items = [] + for index, callback in enumerate(litellm.failure_callback): + if coroutine_checker.is_async_callable(callback): + litellm.logging_callback_manager.add_litellm_async_failure_callback(callback) + removed_async_items.append(index) + elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str): + add_custom_logger_callback_to_specific_event(callback, "failure") + + # Pop the async items from failure_callback in reverse order to avoid index issues + for index in reversed(removed_async_items): + litellm.failure_callback.pop(index) + if callbacks_to_set_up: get_set_callbacks: Final[Callable[[], Callable[..., None]]] = litellm_utils.get_set_callbacks - get_set_callbacks()(callback_list=callback_list, function_id=function_id) - ## ASYNC CALLBACKS - safety net for callbacks added via direct append - if len(litellm.input_callback) > 0: - removed_async_items = [] - for index, callback in enumerate(litellm.input_callback): - if coroutine_checker.is_async_callable(callback): - litellm._async_input_callback.append(callback) - removed_async_items.append(index) - - # Pop the async items from input_callback in reverse order to avoid index issues - for index in reversed(removed_async_items): - litellm.input_callback.pop(index) - if len(litellm.success_callback) > 0: - removed_async_items = [] - for index, callback in enumerate(litellm.success_callback): - if coroutine_checker.is_async_callable(callback): - litellm.logging_callback_manager.add_litellm_async_success_callback(callback) - removed_async_items.append(index) - elif callback == "dynamodb" or callback == "openmeter": - # dynamo is an async callback, it's used for the proxy and needs to be async - # we only support async dynamo db logging for acompletion/aembedding since that's used on proxy - litellm.logging_callback_manager.add_litellm_async_success_callback(callback) - removed_async_items.append(index) - elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str): - add_custom_logger_callback_to_specific_event(callback, "success") - - # Pop the async items from success_callback in reverse order to avoid index issues - for index in reversed(removed_async_items): - litellm.success_callback.pop(index) - - if len(litellm.failure_callback) > 0: - removed_async_items = [] - for index, callback in enumerate(litellm.failure_callback): - if coroutine_checker.is_async_callable(callback): - litellm.logging_callback_manager.add_litellm_async_failure_callback(callback) - removed_async_items.append(index) - elif callback in litellm._known_custom_logger_compatible_callbacks and isinstance(callback, str): - add_custom_logger_callback_to_specific_event(callback, "failure") - - # Pop the async items from failure_callback in reverse order to avoid index issues - for index in reversed(removed_async_items): - litellm.failure_callback.pop(index) + get_set_callbacks()(callback_list=callbacks_to_set_up, function_id=function_id) ### DYNAMIC CALLBACKS ### - dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None - dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None - dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None - dynamic_async_failure_callbacks: Final[list[str | Callable | CustomLogger] | None] = None + dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None + dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None + dynamic_failure_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None + dynamic_async_failure_callbacks: Final[list[str | Callable[..., object] | CustomLogger] | None] = None if kwargs.get("success_callback", None) is not None and isinstance(kwargs["success_callback"], list): removed_async_items = [] for index, callback in enumerate(kwargs["success_callback"]): diff --git a/tests/code_coverage_tests/router_code_coverage.py b/tests/code_coverage_tests/router_code_coverage.py index 55435c51648..af2f386a362 100644 --- a/tests/code_coverage_tests/router_code_coverage.py +++ b/tests/code_coverage_tests/router_code_coverage.py @@ -146,6 +146,18 @@ ignored_function_names = [ "async_admit_deployment_turn", # Bound as the WebSocket turn_admission partial, tested through it in test_router_helper_utils.py "_deployment_as_configured_now", # Read by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py "_admit_turn_through_routing_strategy", # Run by async_admit_deployment_turn, tested through turn_admission in test_router_helper_utils.py + "_callback_selectors", # Run by discard, tested through the discard tests in test_router_helper_utils.py + "bind_to_request_logging", # Run by the proxy user_config route, tested through the user_config tests in test_route_llm_request.py + "serve_through_request_logging", # Run by the proxy user_config route, tested through the user_config tests in test_route_llm_request.py + "_request_adaptive_router_hooks", # Run by serve_through_request_logging, tested through the user_config adaptive test + "_release_global_callbacks", # Run by discard, tested through the discard tests in test_router_helper_utils.py + "_routing_callbacks", # Read by the pre-call checks, tested through the user_config limit tests in test_route_llm_request.py + "_request_policies_not_set_globally", # Read by bind_to_request_logging, tested through the user_config budget test in test_route_llm_request.py + "_optional_loggers", # Read by discard and serve_through_request_logging, tested through the budget limiter discard test + "retire_routing_state", # Run by discard, tested through the budget limiter discard test + "_slack_alerting_loggers", # Run by discard, tested through the Slack alerting discard test in test_router_helper_utils.py + "_adaptive_router_hooks", # Run by discard, tested through the adaptive router discard tests in test_router_helper_utils.py + "_global_callback_candidates", # Run by discard, tested through the survivor discard tests in test_router_helper_utils.py "without_deployments_that_cannot_batch", # Tested through the Azure and OpenAI mixed-group acreate_batch tests in test_router.py "batch_create_routing", # Tested through the mixed-group acreate_batch tests in test_router.py "is_batch_create_routing", # Tested through the chat-after-batch-create test in test_router.py diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index c96e3e2c031..e64f5cca5b8 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -252,6 +252,7 @@ def isolate_host_environment(isolated_aws_config_files: tuple[Path, Path]) -> It @pytest.fixture(autouse=True) def isolate_litellm_globals() -> Iterator[None]: original_callbacks: Final = {name: list(getattr(litellm, name) or []) for name in CALLBACK_LISTS} + original_callback_holders: Final = frozenset(litellm_router_module._routers_holding_global_callbacks) original_reset: Final = {name: getattr(litellm, name) for name in RESET_TO_NONE_GLOBALS} original_restored: Final = {name: getattr(litellm, name) for name in RESTORED_GLOBALS if hasattr(litellm, name)} original_clients: Final = {name: litellm.__dict__[name] for name in MODULE_LEVEL_CLIENTS if name in litellm.__dict__} @@ -263,6 +264,7 @@ def isolate_litellm_globals() -> Iterator[None]: _flush_client_caches() for name in CALLBACK_LISTS: setattr(litellm, name, []) + litellm_router_module._routers_holding_global_callbacks.clear() for name in RESET_TO_NONE_GLOBALS: setattr(litellm, name, None) for name in MODULE_LEVEL_CLIENTS: @@ -277,6 +279,8 @@ def isolate_litellm_globals() -> Iterator[None]: litellm.__dict__.update(original_clients) for name, value in (original_callbacks | original_reset | original_restored).items(): setattr(litellm, name, value) + litellm_router_module._routers_holding_global_callbacks.clear() + litellm_router_module._routers_holding_global_callbacks.update(original_callback_holders) for logger, (level, disabled, propagate, handlers, filters) in original_loggers.items(): logger.setLevel(level) logger.disabled = disabled diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 79645ffe8f6..4f8a13c1182 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -402,6 +402,32 @@ def test_get_combined_callback_list_preserves_insertion_order(logging_obj): ) == ["prometheus", "langfuse", "datadog", "otel", "s3", "gcs_bucket", "arize", "logfire"] +class _InputEventRecorder(CustomLogger): + def __init__(self, label: str, events: list[str]) -> None: + self._label = label + self.events = events + + def log_pre_api_call(self, model: object, messages: object, kwargs: object) -> None: + self.events.append(f"{self._label} pre") + + def log_post_api_call(self, kwargs: object, response_obj: object, start_time: object, end_time: object) -> None: + self.events.append(f"{self._label} post") + + +def test_input_events_reach_the_replacement_and_not_the_global_callback_it_replaced( + logging_obj: LitellmLogging, monkeypatch: pytest.MonkeyPatch +) -> None: + events: Final[list[str]] = [] + monkeypatch.setattr(litellm, "input_callback", [_InputEventRecorder("global", events)]) + logging_obj.replace_global_callback(_InputEventRecorder("replacement", events)) + logging_obj.update_environment_variables(litellm_params={}, optional_params={}) + + logging_obj.pre_call(input="", api_key="") + logging_obj.post_call(original_response="ok", input="", api_key="") + + assert events == ["replacement pre", "replacement post"] + + def test_get_masked_api_base(logging_obj): api_base = "https://api.openai.com/v1" masked_api_base = logging_obj._get_masked_api_base(api_base) diff --git a/tests/unit/litellm_core_utils/test_logging_callback_manager.py b/tests/unit/litellm_core_utils/test_logging_callback_manager.py index f8260702068..25a52137d65 100644 --- a/tests/unit/litellm_core_utils/test_logging_callback_manager.py +++ b/tests/unit/litellm_core_utils/test_logging_callback_manager.py @@ -145,6 +145,7 @@ def test_remove_callback_from_all_lists(): obj = TestLogger() manager.add_litellm_callback(obj) + manager.add_litellm_input_callback(obj) manager.add_litellm_success_callback(obj) manager.add_litellm_failure_callback(obj) manager.add_litellm_async_success_callback(obj) @@ -153,6 +154,7 @@ def test_remove_callback_from_all_lists(): manager.remove_callback_from_all_lists(obj) assert obj not in litellm.callbacks + assert obj not in litellm.input_callback assert obj not in litellm.success_callback assert obj not in litellm.failure_callback assert obj not in litellm._async_success_callback diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 8928b79bcd4..6044475652c 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1,19 +1,42 @@ +import asyncio import contextlib +import gc +import threading +import weakref import httpx import json import litellm import pytest +import redis import respx +from collections.abc import Awaitable, Callable, Coroutine, Iterator, Mapping +from datetime import datetime from types import SimpleNamespace +from uuid import uuid4 from typing import Final from unittest.mock import MagicMock +from fakeredis import TcpFakeServer from fastapi import HTTPException -from litellm.proxy.route_llm_request import JSON_OBJECT_ADAPTER, ProxyModelNotFoundError, route_request -from litellm.types.router import RouterGeneralSettings +from litellm.caching.caching import DualCache +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.route_llm_request import ( + JSON_OBJECT_ADAPTER, + ProxyModelNotFoundError, + UserConfigRouters, + route_request, + user_config_routers, +) +from litellm.proxy.utils import ProxyLogging +from litellm.router_strategy.adaptive_router.config import ( + ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY, + ADAPTIVE_ROUTER_RESPONSE_HEADER, +) +from litellm.types.router import RouterErrors, RouterGeneralSettings, RouterRateLimitError @pytest.mark.parametrize( @@ -2434,3 +2457,571 @@ async def test_team_key_search_of_registered_bedrock_store_reaches_bedrock_with_ posted_url ) assert "Credential=AKIAREGISTRYKEY00000/" in signed_auth, signed_auth + + +@pytest.fixture(autouse=True) +def discard_cached_user_config_routers() -> Iterator[None]: + yield + user_config_routers.clear() + + +@pytest.fixture +def fake_redis_port() -> Iterator[int]: + server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis") + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + yield server.server_address[1] + server.shutdown() + server.server_close() + + +async def _wait_out_a_rate_limit_minute_rollover() -> None: + seconds_left: Final = 60 - datetime.now().second + if seconds_left < 10: + await asyncio.sleep(seconds_left) + + +def _proxy_logging_obj(model: str = "rate-limited") -> LiteLLMLoggingObj: + logging_obj, _ = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + model=model, + messages=[{"role": "user", "content": "hi"}], + litellm_call_id=str(uuid4()), + ) + assert isinstance(logging_obj, LiteLLMLoggingObj) + return logging_obj + + +def _mock_deployment(model_name: str, **deployment_limits: int) -> dict[str, object]: + return { + "model_name": model_name, + **deployment_limits, + "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test", "mock_response": "ok"}, + } + + +async def _user_config_request_outcome( + redis_port: int | None, + routing_settings: Mapping[str, object], + deployment_limits: Mapping[str, int], + logging_obj: LiteLLMLoggingObj | None, +) -> str: + data: Final = { + "model": "rate-limited", + "messages": [{"role": "user", "content": "hi"}], + "num_retries": 0, + **({"litellm_logging_obj": logging_obj} if logging_obj is not None else {}), + "user_config": { + **routing_settings, + **({"redis_host": "127.0.0.1", "redis_port": redis_port} if redis_port is not None else {}), + "model_list": [_mock_deployment("rate-limited", **deployment_limits)], + }, + } + try: + await (await route_request(data, None, None, "acompletion")) + except (litellm.RateLimitError, RouterRateLimitError): + return "429" + finally: + await asyncio.sleep(0.3) + return "200" + + +async def _tpm_for_three_requests() -> int: + messages: Final = [{"role": "user", "content": "hi"}] + probe: Final = await litellm.acompletion(model="openai/gpt-5.5", messages=messages, mock_response="ok") + assert isinstance(probe, litellm.ModelResponse) + probe_usage: Final = getattr(probe, "usage", None) + assert isinstance(probe_usage, litellm.Usage) + return 2 * probe_usage.total_tokens + litellm.token_counter(messages=messages) + + +async def _request_cost() -> float: + probe: Final = await litellm.acompletion( + model="openai/gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok" + ) + return litellm.completion_cost(completion_response=probe) + + +async def _proxy_router_outcomes( + redis_port: int | None, + routing_settings: Mapping[str, object], + deployment_limits: Mapping[str, int], + requests: int, +) -> list[str]: + router: Final = litellm.Router( + **routing_settings, + num_retries=0, + model_list=[_mock_deployment("proxy-rate-limited", **deployment_limits)], + **({"redis_host": "127.0.0.1", "redis_port": redis_port} if redis_port is not None else {}), + ) + + async def outcome() -> str: + try: + await router.acompletion(model="proxy-rate-limited", messages=[{"role": "user", "content": "hi"}]) + except (litellm.RateLimitError, RouterRateLimitError): + return "429" + finally: + await asyncio.sleep(0.3) + return "200" + + try: + return [await outcome() for _ in range(requests)] + finally: + router.discard() + + +@pytest.mark.asyncio +async def test_a_user_config_adaptive_request_answers_with_the_model_it_picked() -> None: + data: Final = { + "model": "smart-cheap-router", + "messages": [{"role": "user", "content": "Write a one line python function that adds two numbers"}], + "metadata": {}, + "user_config": { + "model_list": [ + { + "model_name": "smart-cheap-router", + "litellm_params": { + "model": "auto_router/adaptive_router", + "adaptive_router_config": {"available_models": ["fast", "smart"]}, + }, + }, + { + **_mock_deployment("fast"), + "model_info": {"adaptive_router_preferences": {"quality_tier": 2, "strengths": []}}, + }, + { + **_mock_deployment("smart"), + "model_info": {"adaptive_router_preferences": {"quality_tier": 3, "strengths": ["code_generation"]}}, + }, + ] + }, + } + + await (await route_request(data, None, None, "acompletion")) + headers: Final = await ProxyLogging(user_api_key_cache=DualCache()).post_call_response_headers_hook( + data=data, user_api_key_dict=UserAPIKeyAuth(), response=None + ) + + assert headers == {ADAPTIVE_ROUTER_RESPONSE_HEADER: data["metadata"][ADAPTIVE_ROUTER_CHOSEN_MODEL_KEY]} + + +def _global_router_callbacks() -> tuple[list[object], list[object]]: + return list(litellm.callbacks), list(litellm.input_callback) + + +@pytest.mark.parametrize( + "routing_settings, deployment_limits", + [ + ({"routing_strategy": "usage-based-routing-v2"}, {"rpm": 2}), + ({"routing_strategy": "usage-based-routing-v2"}, {"tpm": 30}), + ({"routing_strategy": "usage-based-routing"}, {"rpm": 2}), + ({"optional_pre_call_checks": ["enforce_model_rate_limits"]}, {"rpm": 2}), + ], +) +@pytest.mark.parametrize("arrives_with_logging_obj", [True, False]) +@pytest.mark.parametrize("with_redis", [True, False]) +@pytest.mark.asyncio +async def test_user_config_router_enforces_its_limit_across_proxy_requests( + fake_redis_port: int, + routing_settings: Mapping[str, object], + deployment_limits: Mapping[str, int], + arrives_with_logging_obj: bool, + with_redis: bool, +) -> None: + await _wait_out_a_rate_limit_minute_rollover() + callbacks_before: Final = _global_router_callbacks() + outcomes: Final = [ + await _user_config_request_outcome( + fake_redis_port if with_redis else None, + routing_settings, + deployment_limits, + _proxy_logging_obj() if arrives_with_logging_obj else None, + ) + for _ in range(6) + ] + + assert "429" in outcomes, outcomes + assert _global_router_callbacks() == callbacks_before + assert outcomes == await _proxy_router_outcomes( + fake_redis_port if with_redis else None, routing_settings, deployment_limits, len(outcomes) + ) + + +@pytest.mark.parametrize("arrives_with_logging_obj", [True, False]) +@pytest.mark.asyncio +async def test_concurrent_user_config_requests_each_count_once_against_the_tpm_limit( + fake_redis_port: int, arrives_with_logging_obj: bool +) -> None: + tpm_for_three_requests: Final = await _tpm_for_three_requests() + + def request() -> Coroutine[None, None, str]: + return _user_config_request_outcome( + fake_redis_port, + {"routing_strategy": "usage-based-routing-v2"}, + {"tpm": tpm_for_three_requests}, + _proxy_logging_obj() if arrives_with_logging_obj else None, + ) + + await _wait_out_a_rate_limit_minute_rollover() + concurrent_outcomes: Final = await asyncio.gather(request(), request()) + later_outcomes: Final = [await request() for _ in range(2)] + + assert [*concurrent_outcomes, *later_outcomes] == ["200", "200", "200", "429"] + + +@pytest.mark.asyncio +async def test_overlapping_user_config_requests_share_one_rpm_limit(fake_redis_port: int) -> None: + def request() -> Coroutine[None, None, str]: + return _user_config_request_outcome( + fake_redis_port, {"routing_strategy": "usage-based-routing-v2"}, {"rpm": 2}, _proxy_logging_obj() + ) + + await _wait_out_a_rate_limit_minute_rollover() + outcomes: Final = await asyncio.gather(*(request() for _ in range(4))) + + assert outcomes.count("200") <= 2, outcomes + assert "429" in outcomes, outcomes + + +@pytest.mark.parametrize( + "routing_strategy, limited_by", + [("usage-based-routing-v2", "rpm"), ("usage-based-routing-v2", "tpm"), ("usage-based-routing", "rpm")], +) +@pytest.mark.parametrize("proxy_router_shares_redis", [True, False]) +@pytest.mark.asyncio +async def test_a_proxy_router_on_the_same_strategy_leaves_the_user_config_limit_unchanged( + fake_redis_port: int, routing_strategy: str, limited_by: str, proxy_router_shares_redis: bool +) -> None: + deployment_limits: Final = {"rpm": 3} if limited_by == "rpm" else {"tpm": await _tpm_for_three_requests()} + + def request() -> Coroutine[None, None, str]: + return _user_config_request_outcome( + fake_redis_port, {"routing_strategy": routing_strategy}, deployment_limits, _proxy_logging_obj() + ) + + async def outcomes() -> list[str]: + first_instance: Final = [await request() for _ in range(2)] + user_config_routers.clear() + second_instance: Final = [await request() for _ in range(3)] + return [*first_instance, *second_instance] + + await _wait_out_a_rate_limit_minute_rollover() + alone: Final = await outcomes() + user_config_routers.clear() + await asyncio.sleep(0.3) + redis.Redis(host="127.0.0.1", port=fake_redis_port).flushall() + proxy_router: Final = litellm.Router( + routing_strategy=routing_strategy, + model_list=[_mock_deployment("proxy-model")], + redis_host="127.0.0.1" if proxy_router_shares_redis else None, + redis_port=fake_redis_port if proxy_router_shares_redis else None, + ) + try: + beside_proxy_router: Final = await outcomes() + finally: + proxy_router.discard() + + assert "429" in alone, alone + assert beside_proxy_router == alone + + +@pytest.mark.parametrize("user_config_settings", [{}, {"optional_pre_call_checks": ["router_budget_limiting"]}]) +@pytest.mark.asyncio +async def test_user_config_requests_stay_under_the_proxy_provider_budget( + user_config_settings: Mapping[str, object], +) -> None: + proxy_router: Final = litellm.Router( + model_list=[_mock_deployment("proxy-model")], + provider_budget_config={"openai": {"budget_limit": await _request_cost() / 2, "time_period": "1d"}}, + ) + try: + await proxy_router.acompletion(model="proxy-model", messages=[{"role": "user", "content": "hi"}]) + await asyncio.sleep(0.3) + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value): + await _user_config_request_outcome(None, user_config_settings, {}, _proxy_logging_obj()) + finally: + proxy_router.discard() + + +@pytest.mark.asyncio +async def test_a_user_config_provider_budget_limits_its_own_requests() -> None: + budget_limit: Final = await _request_cost() / 2 + + def request() -> Coroutine[None, None, str]: + budget: Final = {"openai": {"budget_limit": budget_limit, "time_period": "1d"}} + return _user_config_request_outcome(None, {"provider_budget_config": budget}, {}, _proxy_logging_obj()) + + first: Final = await request() + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value): + await request() + assert first == "200" + + +@pytest.mark.asyncio +async def test_a_request_in_flight_when_its_user_config_router_is_dropped_still_counts_against_its_budget( + fake_redis_port: int, +) -> None: + budget_limit: Final = 1.5 * await _request_cost() + + async def request() -> None: + await ( + await route_request( + { + "model": "budgeted", + "messages": [{"role": "user", "content": "hi"}], + "litellm_logging_obj": _proxy_logging_obj("budgeted"), + "user_config": { + "redis_host": "127.0.0.1", + "redis_port": fake_redis_port, + "provider_budget_config": {"openai": {"budget_limit": budget_limit, "time_period": "1d"}}, + "model_list": [ + { + "model_name": "budgeted", + "litellm_params": { + "model": "openai/gpt-5.5", + "api_key": "sk-test", + "mock_response": "ok", + "mock_delay": 0.5, + }, + } + ], + }, + }, + None, + None, + "acompletion", + ) + ) + + await request() + in_flight: Final = asyncio.create_task(request()) + await asyncio.sleep(0.2) + user_config_routers.clear() + await in_flight + await asyncio.sleep(0.3) + + with pytest.raises(ValueError, match=RouterErrors.no_deployments_with_provider_budget_routing.value): + await request() + + +async def _user_config_request(model: str, user_config: Mapping[str, object], arrives_with_logging_obj: bool) -> object: + return await ( + await route_request( + { + "model": model, + "messages": list(_HELLO_MESSAGES), + "num_retries": 0, + **({"litellm_logging_obj": _proxy_logging_obj(model)} if arrives_with_logging_obj else {}), + "user_config": user_config, + }, + None, + None, + "acompletion", + ) + ) + + +def _ordered_pair(model_name: str, first_base: str, second_base: str) -> list[dict[str, object]]: + return [ + { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test", "api_base": api_base, "order": order}, + } + for order, api_base in ((1, first_base), (2, second_base)) + ] + + +@pytest.mark.parametrize("arrives_with_logging_obj", [True, False]) +@pytest.mark.asyncio +async def test_a_rate_limited_user_config_deployment_cools_down_for_the_next_requests( + monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + rate_limited_base: Final = "https://rate-limited.example/v1" + healthy_base: Final = "https://healthy.example/v1" + user_config: Final = {"model_list": _ordered_pair("flaky", rate_limited_base, healthy_base)} + + with respx.mock(assert_all_called=False) as respx_mock: + rate_limited: Final = respx_mock.post(f"{rate_limited_base}/chat/completions").mock( + return_value=httpx.Response(429, json={"error": {"message": "slow down"}}) + ) + respx_mock.post(f"{healthy_base}/chat/completions").mock(return_value=_chat_completion_response("gpt-5.5")) + for _ in range(3): + await _user_config_request("flaky", user_config, arrives_with_logging_obj) + await asyncio.sleep(0.3) + + assert rate_limited.call_count == 1 + + +async def _remaining_usage_headers_after_429_then_200s( + send: Callable[[], Awaitable[object]], api_base: str +) -> list[tuple[object, object] | str]: + async def remaining_usage() -> tuple[object, object] | str: + try: + response: Final = await send() + except litellm.RateLimitError: + return "429" + finally: + await asyncio.sleep(0.3) + assert isinstance(response, litellm.ModelResponse) + headers: Final = response._hidden_params["additional_headers"] + return headers.get("x-ratelimit-remaining-requests"), headers.get("x-ratelimit-remaining-tokens") + + with respx.mock(assert_all_called=False) as respx_mock: + respx_mock.post(f"{api_base}/chat/completions").mock( + side_effect=[ + httpx.Response(429, json={"error": {"message": "slow down"}}), + _chat_completion_response("gpt-5.5"), + _chat_completion_response("gpt-5.5"), + ] + ) + return [await remaining_usage() for _ in range(3)] + + +@pytest.mark.parametrize("arrives_with_logging_obj", [True, False]) +@pytest.mark.asyncio +async def test_user_config_remaining_usage_headers_match_a_proxy_router_with_the_same_deployment( + monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + await _wait_out_a_rate_limit_minute_rollover() + proxy_base: Final = "https://proxy-limited.example/v1" + user_config_base: Final = "https://user-config-limited.example/v1" + + def deployment(model_name: str, api_base: str) -> dict[str, object]: + return { + "model_name": model_name, + "litellm_params": { + "model": "openai/gpt-5.5", + "api_key": "sk-test", + "api_base": api_base, + "rpm": 10, + "tpm": 1000, + }, + } + + proxy_router: Final = litellm.Router(model_list=[deployment("proxy-limited", proxy_base)]) + try: + proxy_router_headers: Final = await _remaining_usage_headers_after_429_then_200s( + lambda: proxy_router.acompletion(model="proxy-limited", messages=list(_HELLO_MESSAGES), num_retries=0), + proxy_base, + ) + finally: + proxy_router.discard() + + user_config: Final = {"model_list": [deployment("user-config-limited", user_config_base)]} + user_config_headers: Final = await _remaining_usage_headers_after_429_then_200s( + lambda: _user_config_request("user-config-limited", user_config, arrives_with_logging_obj), user_config_base + ) + + assert user_config_headers == proxy_router_headers + + +async def _first_deployment_calls_over_successes_then_failures( + send: Callable[[], Awaitable[object]], first_base: str, second_base: str +) -> int: + successes: Final = [_chat_completion_response("gpt-5.5") for _ in range(6)] + failures: Final = [httpx.Response(500, json={"error": {"message": "boom"}}) for _ in range(6)] + with respx.mock(assert_all_called=False) as respx_mock: + first: Final = respx_mock.post(f"{first_base}/chat/completions").mock(side_effect=[*successes, *failures]) + respx_mock.post(f"{second_base}/chat/completions").mock(return_value=_chat_completion_response("gpt-5.5")) + for _ in range(len(successes) + len(failures)): + await send() + await asyncio.sleep(0.3) + return first.call_count + + +@pytest.mark.parametrize("arrives_with_logging_obj", [True, False]) +@pytest.mark.asyncio +async def test_user_config_successes_keep_a_deployment_out_of_cooldown_like_a_proxy_router( + monkeypatch: pytest.MonkeyPatch, arrives_with_logging_obj: bool +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + await _wait_out_a_rate_limit_minute_rollover() + proxy_router: Final = litellm.Router( + model_list=_ordered_pair("proxy-pair", "https://proxy-first.example/v1", "https://proxy-second.example/v1") + ) + try: + proxy_router_calls: Final = await _first_deployment_calls_over_successes_then_failures( + lambda: proxy_router.acompletion(model="proxy-pair", messages=list(_HELLO_MESSAGES), num_retries=0), + "https://proxy-first.example/v1", + "https://proxy-second.example/v1", + ) + finally: + proxy_router.discard() + + user_config: Final = { + "model_list": _ordered_pair("user-pair", "https://user-first.example/v1", "https://user-second.example/v1") + } + + user_config_calls: Final = await _first_deployment_calls_over_successes_then_failures( + lambda: _user_config_request("user-pair", user_config, arrives_with_logging_obj), + "https://user-first.example/v1", + "https://user-second.example/v1", + ) + + assert user_config_calls == proxy_router_calls + + +@pytest.mark.asyncio +async def test_user_config_adaptive_router_learns_from_proxy_requests() -> None: + user_config: Final = { + "model_list": [ + _mock_deployment("smart"), + _mock_deployment("fast"), + { + "model_name": "adaptive", + "litellm_params": { + "model": "auto_router/adaptive_router", + "adaptive_router_config": {"available_models": ["smart", "fast"]}, + }, + }, + ] + } + await ( + await route_request( + { + "model": "adaptive", + "messages": [{"role": "user", "content": "write a sorting function"}], + "metadata": {"session_id": "session-1"}, + "litellm_logging_obj": _proxy_logging_obj("adaptive"), + "user_config": user_config, + }, + None, + None, + "acompletion", + ) + ) + await asyncio.sleep(0.3) + + router: Final = user_config_routers.get(user_config, lambda: pytest.fail("the user_config router was not reused")) + snapshot: Final = await router.adaptive_routers["adaptive"][0].strategy.get_state_snapshot() + assert snapshot["feedback_contexts_live"] == 1, snapshot + + +@pytest.mark.asyncio +async def test_user_config_routers_discard_the_least_recently_used_router_past_capacity() -> None: + routers: Final = UserConfigRouters(max_size=2) + + def config(model_name: str) -> dict[str, object]: + return {"routing_strategy": "usage-based-routing-v2", "model_list": [_mock_deployment(model_name)]} + + def build(model_name: str) -> litellm.Router: + return litellm.Router(**config(model_name)) + + def must_be_cached() -> litellm.Router: + pytest.fail("a cached router was rebuilt") + + first_handler: Final = weakref.ref(routers.get(config("first"), lambda: build("first")).lowesttpm_logger_v2) + second_handler: Final = weakref.ref(routers.get(config("second"), lambda: build("second")).lowesttpm_logger_v2) + routers.get(config("first"), must_be_cached) + routers.get(config("third"), lambda: build("third")) + await asyncio.sleep(0.2) + gc.collect() + + assert routers.get(config("first"), must_be_cached) is not None + assert first_handler() is not None + assert second_handler() is None + routers.clear() diff --git a/tests/unit/router_strategy/adaptive_router/test_hooks.py b/tests/unit/router_strategy/adaptive_router/test_hooks.py index ad61f43c5a0..0ed22658144 100644 --- a/tests/unit/router_strategy/adaptive_router/test_hooks.py +++ b/tests/unit/router_strategy/adaptive_router/test_hooks.py @@ -10,6 +10,7 @@ from litellm.router_strategy.adaptive_router.config import ( ) from litellm.router_strategy.adaptive_router.hooks import ( AdaptiveRouterPostCallHook, + AdaptiveRouterResponseHeaderHook, _recent_tool_results, _resolve_session_key, ) @@ -337,8 +338,7 @@ async def test_post_call_response_headers_hook_returns_chosen_model_header(): """The header hook returns the `x-litellm-adaptive-router-model` header so proxy header construction picks it up (works for both streaming and non-streaming; `async_post_call_success_hook` is too late for streaming).""" - hook = _make_hook() - headers = await hook.async_post_call_response_headers_hook( + headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook( data={"metadata": {"adaptive_router_chosen_model": "smart"}}, user_api_key_dict=MagicMock(), response=MagicMock(), @@ -348,8 +348,7 @@ async def test_post_call_response_headers_hook_returns_chosen_model_header(): @pytest.mark.asyncio async def test_post_call_response_headers_hook_noop_when_metadata_missing_key(): - hook = _make_hook() - headers = await hook.async_post_call_response_headers_hook( + headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook( data={"metadata": {"litellm_session_id": "sess-A"}}, user_api_key_dict=MagicMock(), response=MagicMock(), @@ -359,8 +358,7 @@ async def test_post_call_response_headers_hook_noop_when_metadata_missing_key(): @pytest.mark.asyncio async def test_post_call_response_headers_hook_noop_when_no_metadata(): - hook = _make_hook() - headers = await hook.async_post_call_response_headers_hook( + headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook( data={}, user_api_key_dict=MagicMock(), response=MagicMock(), @@ -370,8 +368,7 @@ async def test_post_call_response_headers_hook_noop_when_no_metadata(): @pytest.mark.asyncio async def test_post_call_response_headers_hook_noop_when_metadata_not_dict(): - hook = _make_hook() - headers = await hook.async_post_call_response_headers_hook( + headers = await AdaptiveRouterResponseHeaderHook().async_post_call_response_headers_hook( data={"metadata": "not-a-dict"}, user_api_key_dict=MagicMock(), response=MagicMock(), diff --git a/tests/unit/router_strategy/test_base_routing_strategy.py b/tests/unit/router_strategy/test_base_routing_strategy.py index dc75c3d2916..19ff11dd3f6 100644 --- a/tests/unit/router_strategy/test_base_routing_strategy.py +++ b/tests/unit/router_strategy/test_base_routing_strategy.py @@ -1,15 +1,20 @@ import json import logging -from typing import Any, Dict, List, Optional, Set, Union +from typing import Any, Dict, Final, List, Optional, Set, Union import pytest import asyncio +import threading from unittest.mock import MagicMock, patch +from fakeredis import TcpFakeServer + from litellm.caching.caching import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.caching.redis_cache import RedisCache from litellm.caching.redis_cache import RedisCircuitBreakerOpenError, RedisPipelineIncrementOperation from litellm.router_strategy.base_routing_strategy import BaseRoutingStrategy @@ -162,3 +167,24 @@ async def test_push_refused_by_the_open_circuit_breaker_is_not_logged_as_an_erro assert caplog.records == [] assert base_strategy.redis_increment_operation_queue == [] + + +@pytest.mark.asyncio +async def test_an_increment_counted_after_retire_still_reaches_redis(): + server: Final = TcpFakeServer(("127.0.0.1", 0), server_type="redis") + threading.Thread(target=server.serve_forever, daemon=True).start() + redis_cache: Final = RedisCache(host="127.0.0.1", port=server.server_address[1]) + strategy: Final = BaseRoutingStrategy( + dual_cache=DualCache(redis_cache=redis_cache, in_memory_cache=InMemoryCache()), + should_batch_redis_writes=True, + default_sync_interval=60, + ) + try: + strategy.retire() + await strategy._increment_value_in_current_window("counted-after-retire", 3, 60) + await asyncio.sleep(0.2) + + assert float(await redis_cache.async_get_cache("counted-after-retire")) == 3 + finally: + server.shutdown() + server.server_close() diff --git a/tests/unit/router_strategy/test_budget_limiter_hotpath.py b/tests/unit/router_strategy/test_budget_limiter_hotpath.py index 4667ac67efa..10704a29347 100644 --- a/tests/unit/router_strategy/test_budget_limiter_hotpath.py +++ b/tests/unit/router_strategy/test_budget_limiter_hotpath.py @@ -1,4 +1,5 @@ import asyncio +import contextlib import gc import logging from types import SimpleNamespace @@ -492,6 +493,8 @@ def _new_router_budget_limiter( budget_limiter._redis_increment_queue_lock = queue_lock if queue_lock is not None else asyncio.Lock() budget_limiter._redis_increment_flush_lock = asyncio.Lock() budget_limiter._detached_increment_operations = None + budget_limiter._retired = False + budget_limiter._flush_after_retire = None return budget_limiter @@ -784,3 +787,57 @@ async def test_cancelled_flush_does_not_requeue_an_applied_batch(cancellations: await limiter._push_in_memory_increments_to_redis() assert redis_cache.values[_SPEND_KEY] == 10.0 assert limiter.redis_increment_operation_queue == [] + + +@pytest.mark.asyncio +async def test_spend_queued_when_a_limiter_is_retired_reaches_redis_after_a_failed_write() -> None: + pipeline_started: Final = asyncio.Event() + pipeline_completed: Final = asyncio.Event() + redis_cache: Final = _MockRedisCache( + initial_values={}, + pipeline_started=pipeline_started, + pipeline_completed=pipeline_completed, + should_fail_pipeline=True, + ) + budget_limiter: Final = _new_router_budget_limiter( + redis_cache=redis_cache, redis_increment_operation_queue=[_increment(10.0)] + ) + budget_limiter._sync_task = asyncio.create_task(asyncio.sleep(3600)) + + budget_limiter.retire() + await asyncio.wait_for(pipeline_started.wait(), timeout=1) + assert redis_cache.values == {} + redis_cache.should_fail_pipeline = False + await asyncio.wait_for(pipeline_completed.wait(), timeout=5) + + assert redis_cache.values == {_SPEND_KEY: 10.0} + assert budget_limiter.redis_increment_operation_queue == [] + + +@pytest.mark.asyncio +async def test_spend_in_flight_when_a_limiter_is_retired_reaches_redis_after_that_write_fails() -> None: + pipeline_started: Final = asyncio.Event() + allow_pipeline_to_complete: Final = asyncio.Event() + pipeline_completed: Final = asyncio.Event() + redis_cache: Final = _MockRedisCache( + initial_values={}, + pipeline_started=pipeline_started, + allow_pipeline_to_complete=allow_pipeline_to_complete, + should_fail_pipeline=True, + pipeline_completed=pipeline_completed, + ) + budget_limiter: Final = _new_router_budget_limiter( + redis_cache=redis_cache, redis_increment_operation_queue=[_increment(10.0)] + ) + budget_limiter._sync_task = asyncio.create_task(budget_limiter.periodic_sync_in_memory_spend_with_redis()) + await asyncio.wait_for(pipeline_started.wait(), timeout=1) + + budget_limiter.retire() + allow_pipeline_to_complete.set() + with contextlib.suppress(asyncio.CancelledError): + await budget_limiter._sync_task + redis_cache.should_fail_pipeline = False + await asyncio.wait_for(pipeline_completed.wait(), timeout=5) + + assert redis_cache.values == {_SPEND_KEY: 10.0} + assert budget_limiter.redis_increment_operation_queue == [] diff --git a/tests/unit/test_router/test_router_helper_utils.py b/tests/unit/test_router/test_router_helper_utils.py index 7088818aa01..af8e7f53462 100644 --- a/tests/unit/test_router/test_router_helper_utils.py +++ b/tests/unit/test_router/test_router_helper_utils.py @@ -1,8 +1,12 @@ +import asyncio import functools +import gc import json import os +import weakref from datetime import datetime, timezone -from collections.abc import Mapping +from collections.abc import Iterator, Mapping +from types import MethodType from typing import Final from unittest.mock import AsyncMock, MagicMock, patch @@ -12,8 +16,12 @@ from litellm import Router from litellm.caching.dual_cache import DualCache from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY +from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo +from litellm.router_strategy.adaptive_router.hooks import AdaptiveRouterPostCallHook +from litellm.types.litellm_params import RoutingStrategyName +from litellm.types.router import AlertingConfig, Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo from litellm.types.utils import ( CallTypes, ModelResponse, @@ -1852,6 +1860,315 @@ def test_discard(model_list): assert len(litellm.service_callback) == 0 +_CALLBACK_LIST_NAMES: Final = ( + "callbacks", + "input_callback", + "success_callback", + "failure_callback", + "_async_success_callback", + "_async_failure_callback", +) + + +def _callback_ids(list_name: str) -> list[int]: + return [id(callback) for callback in getattr(litellm, list_name)] + + +@pytest.mark.parametrize( + "routing_strategy, selector_attr", + [ + ("least-busy", "leastbusy_logger"), + ("usage-based-routing", "lowesttpm_logger"), + ("usage-based-routing-v2", "lowesttpm_logger_v2"), + ("latency-based-routing", "lowestlatency_logger"), + ("cost-based-routing", "lowestcost_logger"), + ], +) +@pytest.mark.asyncio +async def test_discard_releases_strategy_selector_across_router_lifecycles( + routing_strategy: RoutingStrategyName, selector_attr: str +): + def retire() -> weakref.ReferenceType[object]: + router: Final = Router(model_list=[], routing_strategy=routing_strategy) + selector: Final = getattr(router, selector_attr) + assert any(callback is selector for callback in litellm.callbacks) + router.discard() + return weakref.ref(selector) + + selector_refs: Final = tuple(retire() for _ in range(3)) + await asyncio.sleep(0) + gc.collect() + + assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES) + assert all(selector_ref() is None for selector_ref in selector_refs) + + +@pytest.mark.parametrize( + "routing_strategy, selector_attr", + [ + ("least-busy", "leastbusy_logger"), + ("usage-based-routing", "lowesttpm_logger"), + ("usage-based-routing-v2", "lowesttpm_logger_v2"), + ("latency-based-routing", "lowestlatency_logger"), + ("cost-based-routing", "lowestcost_logger"), + ], +) +@pytest.mark.asyncio +async def test_discard_purges_selector_from_event_lists_after_serving_a_request( + routing_strategy: RoutingStrategyName, selector_attr: str +): + router: Final = Router( + model_list=[{"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}], + routing_strategy=routing_strategy, + ) + selector: Final = getattr(router, selector_attr) + await router.acompletion(model="gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok") + assert any(callback is selector for callback in litellm.success_callback) + + router.discard() + + assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES) + + +def test_discard_keeps_callbacks_owned_by_other_live_routers(): + survivor: Final = Router(model_list=[], routing_strategy="latency-based-routing") + unrelated: Final = CustomLogger() + litellm.logging_callback_manager.add_litellm_callback(unrelated) + before_ids: Final = {list_name: _callback_ids(list_name) for list_name in _CALLBACK_LIST_NAMES} + retired: Final = Router(model_list=[], routing_strategy="least-busy") + assert any(callback is retired.leastbusy_logger for callback in litellm.callbacks) + + retired.discard() + + assert {list_name: _callback_ids(list_name) for list_name in before_ids} == before_ids + assert any(callback is survivor.lowestlatency_logger for callback in litellm.callbacks) + survivor.discard() + + +@pytest.mark.parametrize( + "routing_strategy, selector_attr", + [ + ("usage-based-routing", "lowesttpm_logger"), + ("usage-based-routing-v2", "lowesttpm_logger_v2"), + ("latency-based-routing", "lowestlatency_logger"), + ("cost-based-routing", "lowestcost_logger"), + ], +) +@pytest.mark.asyncio +async def test_discard_leaves_same_strategy_survivor_receiving_events( + routing_strategy: RoutingStrategyName, selector_attr: str +): + model_list: Final[list[DeploymentTypedDict]] = [ + {"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}} + ] + retired: Final = Router(model_list=model_list, routing_strategy=routing_strategy) + survivor: Final = Router(model_list=model_list, routing_strategy=routing_strategy) + + retired.discard() + await survivor.acompletion(model="gpt-5.5", messages=[{"role": "user", "content": "hi"}], mock_response="ok") + + assert any(callback is getattr(survivor, selector_attr) for callback in litellm.success_callback) + survivor.discard() + + +def test_discard_drops_routing_group_selectors(): + router: Final = Router( + model_list=[ + {"model_name": "grouped-model", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}} + ], + routing_groups=[ + {"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "latency-based-routing"} + ], + ) + _, group_selector = router._get_routing_context("grouped-model") + assert any(callback is group_selector for callback in litellm.callbacks) + + router.discard() + + assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES) + + +_GROUPED_MODEL_LIST: Final[list[DeploymentTypedDict]] = [ + {"model_name": "grouped-model", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}} +] + + +class _LabelledHook(CustomLogger): + def __init__(self, label: str) -> None: + super().__init__() + self.label = label + + +def _visit_hooks_reloading_routing_groups_mid_loop( + router: Router, hooks: tuple[_LabelledHook, ...] +) -> Iterator[CustomLogger]: + for callback in litellm.callbacks: + if callback is hooks[0]: + router.update_settings( + routing_groups=[{"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "least-busy"}] + ) + if callback in hooks: + yield callback + + +def test_routing_group_reload_keeps_every_hook_of_an_in_flight_callback_loop(): + router: Final = Router( + model_list=_GROUPED_MODEL_LIST, + routing_groups=[{"group_name": "fast", "models": ["grouped-model"], "routing_strategy": "least-busy"}], + ) + hooks: Final = tuple(_LabelledHook(label) for label in ("guardrail_a", "guardrail_b", "guardrail_c")) + for hook in hooks: + litellm.logging_callback_manager.add_litellm_callback(hook) + + visited: Final = tuple(_visit_hooks_reloading_routing_groups_mid_loop(router, hooks)) + + assert visited == hooks + router.discard() + + +def _routers_seen_in_success_loop_discarding_first(first: Router, second: Router) -> Iterator[Router]: + for callback in litellm.success_callback: + if not isinstance(callback, MethodType): + continue + if callback.__self__ is first: + first.discard() + yield first + if callback.__self__ is second: + yield second + + +def test_discard_mid_success_loop_keeps_the_next_router_callback(): + first: Final = Router(model_list=_GROUPED_MODEL_LIST) + second: Final = Router(model_list=_GROUPED_MODEL_LIST) + + seen: Final = tuple(_routers_seen_in_success_loop_discarding_first(first, second)) + + assert seen == (first, second) + second.discard() + + +@pytest.mark.asyncio +async def test_discard_purges_optional_pre_call_checks_from_event_lists_after_serving_a_request(): + router: Final = Router( + model_list=_GROUPED_MODEL_LIST, + optional_pre_call_checks=["enforce_model_rate_limits", "prompt_caching", "deployment_affinity"], + ) + await router.acompletion(model="grouped-model", messages=[{"role": "user", "content": "hi"}], mock_response="ok") + + router.discard() + + assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES) + + +def test_discard_registers_same_config_survivor_optional_pre_call_checks(): + retired: Final = Router(model_list=_GROUPED_MODEL_LIST, optional_pre_call_checks=["enforce_model_rate_limits"]) + survivor: Final = Router(model_list=_GROUPED_MODEL_LIST, optional_pre_call_checks=["enforce_model_rate_limits"]) + + retired.discard() + + assert survivor.optional_callbacks + assert all(_is_registered(check) for check in survivor.optional_callbacks) + survivor.discard() + + +def _is_registered(callback: object) -> bool: + return any(registered is callback for registered in litellm.callbacks) + + +@pytest.mark.asyncio +async def test_discard_releases_per_request_override_selector(): + def build_override_then_discard() -> weakref.ReferenceType[object]: + router: Final = Router(model_list=_GROUPED_MODEL_LIST) + selector: Final = router._get_override_strategy_selector("usage-based-routing-v2") + assert selector is not None + router.discard() + return weakref.ref(selector) + + selector_ref: Final = build_override_then_discard() + await asyncio.sleep(0) + gc.collect() + + assert selector_ref() is None + + + +def test_discard_registers_the_selector_of_a_same_strategy_survivor_built_without_a_model_list(): + retired: Final = Router(model_list=[], routing_strategy="latency-based-routing") + survivor: Final = Router(routing_strategy="latency-based-routing") + assert not _is_registered(survivor.lowestlatency_logger) + + retired.discard() + + assert _is_registered(survivor.lowestlatency_logger) + survivor.discard() + + +def test_discard_drops_its_slack_alerting_callbacks(): + router: Final = Router(model_list=_GROUPED_MODEL_LIST, alerting_config=AlertingConfig(webhook_url="test")) + assert any(isinstance(callback, SlackAlerting) for callback in litellm.callbacks) + + router.discard() + + assert all(getattr(litellm, list_name) == [] for list_name in _CALLBACK_LIST_NAMES) + + +def _adaptive_router_model_list(name: str) -> list[DeploymentTypedDict]: + return [ + {"model_name": "gpt-5.5", "litellm_params": {"model": "openai/gpt-5.5", "api_key": "sk-test"}}, + { + "model_name": name, + "litellm_params": { + "model": "auto_router/adaptive_router", + "adaptive_router_config": {"available_models": ["gpt-5.5"]}, + }, + }, + ] + + +def _registered_adaptive_router_strategies() -> tuple[object, ...]: + return tuple( + callback.adaptive_router for callback in litellm.callbacks if isinstance(callback, AdaptiveRouterPostCallHook) + ) + + +def test_discard_drops_its_adaptive_router_hook(): + router: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True) + assert _registered_adaptive_router_strategies() + + router.discard() + + assert _registered_adaptive_router_strategies() == () + + +def test_discard_registers_the_adaptive_router_hook_of_a_survivor(): + survivor: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True) + retired: Final = Router(model_list=_adaptive_router_model_list("adaptive"), ignore_invalid_deployments=True) + + retired.discard() + + survivor_strategies: Final = tuple(tagged.strategy for tagged in survivor.adaptive_routers["adaptive"]) + assert _registered_adaptive_router_strategies() == survivor_strategies + survivor.discard() + + +@pytest.mark.asyncio +async def test_discard_releases_its_router_budget_limiter(): + def build_budget_limiter_then_discard() -> weakref.ReferenceType[object]: + router: Final = Router( + model_list=_GROUPED_MODEL_LIST, + provider_budget_config={"openai": {"budget_limit": 10.0, "time_period": "1d"}}, + ) + budget_limiter: Final = router.get_router_deployment_budget_limiter() + assert budget_limiter is not None + router.discard() + return weakref.ref(budget_limiter) + + budget_limiter_ref: Final = build_budget_limiter_then_discard() + await asyncio.sleep(0) + gc.collect() + + assert budget_limiter_ref() is None + def test_initialize_assistants_endpoint(model_list): """Test if the 'initialize_assistants_endpoint' function is working correctly""" router = Router(model_list=model_list) diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 9b6700e9d39..cdff03af3b4 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -3891,6 +3891,114 @@ class TestCallbackAsyncSyncSeparation: assert my_sync_cb not in litellm._async_input_callback +class _DropsACallbackFromAnotherThreadWhenComparedToDynamodb: + def __init__(self, dropped: Callable[..., object]) -> None: + self.dropped = dropped + self.drop_thread: threading.Thread | None = None + + def __eq__(self, other: object) -> bool: + if isinstance(other, str) and other == "dynamodb" and self.drop_thread is None: + self.drop_thread = threading.Thread( + target=litellm.logging_callback_manager.drop_callbacks, args=((self.dropped,),) + ) + self.drop_thread.start() + self.drop_thread.join(timeout=0.2) + return self is other + + __hash__ = object.__hash__ + + +def test_function_setup_moves_async_callbacks_while_another_thread_drops_a_callback( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def dropped_callback(*args: object, **kwargs: object) -> None: + return None + + async def async_callback(*args: object, **kwargs: object) -> None: + return None + + drops_on_compare: Final = _DropsACallbackFromAnotherThreadWhenComparedToDynamodb(dropped_callback) + monkeypatch.setattr(litellm.utils, "callback_list", ["initialized"]) + for list_name in ( + "callbacks", + "input_callback", + "failure_callback", + "service_callback", + "_async_input_callback", + "_async_success_callback", + "_async_failure_callback", + ): + monkeypatch.setattr(litellm, list_name, []) + monkeypatch.setattr(litellm, "success_callback", [dropped_callback, drops_on_compare, async_callback]) + + litellm.utils.function_setup( + "completion", + litellm.utils.Rules(), + datetime.now(), + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + litellm_call_id="callback-drop-race", + ) + assert drops_on_compare.drop_thread is not None + drops_on_compare.drop_thread.join() + + assert litellm.success_callback == [drops_on_compare] + assert litellm._async_success_callback == [async_callback] + + +class _DropsACallbackFromAnotherThreadWhenComparedToSentry: + def __init__(self, dropped: Callable[..., object]) -> None: + self.dropped = dropped + self.drop_thread: threading.Thread | None = None + self.drop_finished_during_logger_setup = False + + def __eq__(self, other: object) -> bool: + if isinstance(other, str) and other == "sentry" and self.drop_thread is None: + self.drop_thread = threading.Thread( + target=litellm.logging_callback_manager.drop_callbacks, args=((self.dropped,),) + ) + self.drop_thread.start() + self.drop_thread.join(timeout=5) + self.drop_finished_during_logger_setup = not self.drop_thread.is_alive() + return self is other + + __hash__ = object.__hash__ + + +def test_function_setup_lets_other_threads_drop_callbacks_while_it_sets_up_loggers( + monkeypatch: pytest.MonkeyPatch, +) -> None: + def dropped_callback(*args: object, **kwargs: object) -> None: + return None + + drops_on_compare: Final = _DropsACallbackFromAnotherThreadWhenComparedToSentry(dropped_callback) + monkeypatch.setattr(litellm.utils, "callback_list", []) + for list_name in ( + "callbacks", + "input_callback", + "failure_callback", + "service_callback", + "_async_input_callback", + "_async_success_callback", + "_async_failure_callback", + ): + monkeypatch.setattr(litellm, list_name, []) + monkeypatch.setattr(litellm, "success_callback", [drops_on_compare]) + + litellm.utils.function_setup( + "completion", + litellm.utils.Rules(), + datetime.now(), + model="gpt-4o", + messages=[{"role": "user", "content": "hi"}], + litellm_call_id="logger-setup-outside-lock", + ) + assert drops_on_compare.drop_thread is not None + drops_on_compare.drop_thread.join() + + assert drops_on_compare.drop_finished_during_logger_setup + + class TestMetadataNoneHandling: """ Test that metadata=None in kwargs doesn't cause TypeError. From 11b5d1a8171148d9dac139652650b2aa5d1dd646 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 02:42:48 +0000 Subject: [PATCH 044/235] feat(gemini): set the long decode budget for requests with no output limit from LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS (#47542) * feat(gemini): LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS sets the long decode budget for requests that send no limit * test(gemini): cover a default long decode budget set by LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS * fix(gemini): apply the default output budget only to requests asking for text output * refactor(gemini): apply the default output budget once in the async request transform * fix(gemini): leave the default output budget off non-chat models and a snake_case generation_config * fix(gemini): skip the default output budget for audio-only models and continuation resumes Lyria and other Gemini chat models whose cost map row lists no text output answer 400 when sent a maxOutputTokens, and a request that resumes from a continuation token answers 400 when a limit is added beside it. Both now go out without the default budget. * fix(vertex_ai): skip the Gemini default budget for a model a deployment's model_info overrides, and pass odd generationConfig shapes through * fix(vertex_ai): cap the Gemini default budget per deployment at the loaded cost map and its own model_info limit * test(vertex_ai): wrap a long test signature under the 120 character limit * fix(vertex_ai): keep a direct call's default output budget under the loaded cost map limit A litellm.Router deployment's model_info writes onto the shared cost map row, so a later direct call with no model_info read that raised value and Vertex answered 400. The direct call now caps at the lower of the loaded row and the live row, so litellm.register_model can still lower the cap. * fix(vertex_ai): ignore a deployment output limit that is not positive A model_info.max_output_tokens of 0 or -1 capped the default budget at that value, and Google answers 400 to a limit that is not positive. It now counts as unreadable, so the cost map limit applies. * fix(gemini): never continue a request asking for more than one candidate in camelCase * fix(gemini): keep continuing a request whose candidate count is null or 0 --- litellm/llms/vertex_ai/gemini/long_decode.py | 51 +- .../llms/vertex_ai/gemini/transformation.py | 114 +++- .../llms/vertex_ai/gemini/test_long_decode.py | 510 +++++++++++++++++- 3 files changed, 647 insertions(+), 28 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/long_decode.py b/litellm/llms/vertex_ai/gemini/long_decode.py index 81d6c04d0e8..571fa94d4a5 100644 --- a/litellm/llms/vertex_ai/gemini/long_decode.py +++ b/litellm/llms/vertex_ai/gemini/long_decode.py @@ -2,17 +2,20 @@ import json from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass from itertools import accumulate, chain -from typing import Final, TypeAlias, TypeVar, cast # noqa: TID251 # sum_usage_metadata casts computed keys +from typing import Final, Literal, TypeAlias, TypeVar, cast # noqa: TID251 # sum_usage_metadata casts computed keys import httpx +from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.exceptions import Timeout +from litellm.litellm_core_utils.env_utils import get_env_int_or_none from litellm.types.llms.vertex_ai import ( Candidates, Citation, CitationMetadata, GenerateContentResponseBody, + GenerationConfig, GroundingMetadata, GroundingSegment, GroundingSupport, @@ -48,8 +51,14 @@ HOP_FAILURES: Final = (VertexAIError, httpx.HTTPError, Timeout) HOP_END_FINISH_REASON: Final = "CONTINUATION" CUT_OFF_FINISH_REASONS: Final = frozenset({"MAX_TOKENS", HOP_END_FINISH_REASON}) +DEFAULT_OUTPUT_BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS" +_OUTPUT_LIMIT_KEYS: Final = frozenset({"max_output_tokens", "maxOutputTokens"}) +_REQUEST_KEYS_THAT_SKIP_THE_DEFAULT_BUDGET: Final = frozenset({"generation_config", "continuationToken"}) +_TEXT_ONLY_MODALITIES: Final = TypeAdapter(list[Literal["TEXT"]]) + _VERTEX_V1_PATH: Final = "/v1/projects/" _VERTEX_PATH_ACCEPTING_CONTINUATION_TOKEN: Final = "/v1beta1/projects/" +_CANDIDATE_COUNT_KEYS: Final = ("candidate_count", "candidateCount") LongDecodeHops: TypeAlias = tuple[GenerateContentResponseBody, ...] AsyncHopPost: TypeAlias = Callable[[RequestBody], Awaitable[httpx.Response]] @@ -76,15 +85,53 @@ def _int_or_none(value: object) -> int | None: return value if isinstance(value, int) else None +def _asks_for_one_candidate(generation_config: Mapping[str, object]) -> bool: + return all( + _int_or_none(generation_config[key] or 1) == 1 for key in _CANDIDATE_COUNT_KEYS if key in generation_config + ) + + def continuation_cap_of(request_body: RequestBody) -> int | None: generation_config: Final = request_body.get("generationConfig") if generation_config is None or "max_output_tokens" not in generation_config: return None - if "candidate_count" in generation_config and _int_or_none(generation_config["candidate_count"]) != 1: + if not _asks_for_one_candidate(generation_config): return None return _int_or_none(generation_config["max_output_tokens"]) +def _asks_for_text_only(generation_config: Mapping[str, object]) -> bool: + if "response_modalities" in generation_config: + return False + try: + _ = _TEXT_ONLY_MODALITIES.validate_python(generation_config.get("responseModalities", [])) + except ValidationError: + return False + return True + + +def with_default_output_budget( + request_body: RequestBody, model_max_output_tokens: Callable[[], int | None] +) -> RequestBody: + generation_config: Final = request_body.get("generationConfig", GenerationConfig()) + request_json: Final[Mapping[str, object]] = request_body + budget: Final = get_env_int_or_none(DEFAULT_OUTPUT_BUDGET_ENV) + if budget is None or budget <= 0: + return request_body + if not _REQUEST_KEYS_THAT_SKIP_THE_DEFAULT_BUDGET.isdisjoint(request_body): + return request_body + if not isinstance(request_json.get("generationConfig", {}), dict): + return request_body + if not _OUTPUT_LIMIT_KEYS.isdisjoint(generation_config): + return request_body + if not _asks_for_text_only(generation_config): + return request_body + model_max: Final = model_max_output_tokens() + if model_max is None: + return request_body + return {**request_body, "generationConfig": {**generation_config, "max_output_tokens": min(budget, model_max)}} + + def client_continuation_token(candidate: Candidates) -> str | None: if candidate.get("finishReason") not in CUT_OFF_FINISH_REASONS: return None diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index e5ce395031a..71bb05ee155 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -7,16 +7,19 @@ Why separate file? Make it easy to see how transformation works import os import re from collections.abc import Callable, Mapping +from functools import partial from typing import TYPE_CHECKING, Any, Final, Literal, cast from urllib.parse import quote import httpx -from pydantic import BaseModel, TypeAdapter +from pydantic import BaseModel, PositiveInt, TypeAdapter, ValidationError +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import drops_unsupported_params +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_image_mime_type_from_url, ) @@ -65,6 +68,7 @@ from ..common_utils import ( get_supports_response_schema, get_supports_system_message, ) +from .long_decode import with_default_output_budget # VertexBase is imported only for type checking to avoid introducing a # module-load-time cyclic import to vertex_llm_base. The instance is lazily @@ -1304,13 +1308,75 @@ def sync_transform_request_body( vertex_auth_header=vertex_auth_header, ) - return transform_request_body( - messages=messages, - model=model, - custom_llm_provider=custom_llm_provider, - litellm_params=litellm_params, - cached_content=cached_content, - optional_params=optional_params, + return _with_default_output_budget( + transform_request_body( + messages=messages, + model=model, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + cached_content=cached_content, + optional_params=optional_params, + ), + model, + custom_llm_provider, + litellm_params.get("model_info"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is an untyped dict + ) + + +class _CostMapOutputModalities(TypedDict): + supported_output_modalities: NotRequired[ReadOnly[tuple[str, ...]]] + + +_COST_MAP_OUTPUT_MODALITIES: Final = TypeAdapter(_CostMapOutputModalities) + + +def _answers_text(cost_map_key: str) -> bool: + try: + row: Final = _COST_MAP_OUTPUT_MODALITIES.validate_python(litellm.model_cost.get(cost_map_key)) # pyright: ignore[reportUnknownMemberType] # cost map rows are untyped JSON + except ValidationError: + return False + return "text" in row.get("supported_output_modalities", ("text",)) + + +class _OutputLimit(TypedDict): + max_output_tokens: NotRequired[ReadOnly[PositiveInt | None]] + + +_OUTPUT_LIMIT: Final = TypeAdapter(_OutputLimit) + + +def _output_limit(fields: object) -> int | None: + try: + return _OUTPUT_LIMIT.validate_python(fields).get("max_output_tokens") + except ValidationError: + return None + + +def _catalog_capped_output_limit(cost_map_key: str, override: object) -> int | None: + limits: Final = ( + _output_limit(GetModelCostMap.loaded_model_cost_map().get(cost_map_key)), + _output_limit(override), + ) + return min((limit for limit in limits if limit is not None), default=None) + + +def _text_chat_model_max_output_tokens( + model: str, custom_llm_provider: str, deployment_model_info: object +) -> int | None: + info: Final = litellm.utils.lookup_model_info_for_registration(model, custom_llm_provider) + if isinstance(info, litellm.utils.ModelInfoLookupMiss) or info.get("mode") != "chat": + return None + if not _answers_text(info["key"]): + return None + return _catalog_capped_output_limit(info["key"], deployment_model_info or info) + + +def _with_default_output_budget( + request_body: RequestBody, model: str, custom_llm_provider: str, deployment_model_info: object +) -> RequestBody: + return with_default_output_budget( + request_body, + partial(_text_chat_model_max_output_tokens, model, custom_llm_provider, deployment_model_info), ) @@ -1380,21 +1446,8 @@ async def async_transform_request_body( messages, should_inline=should_inline_remote_media(custom_llm_provider) ) - if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages): - # _transform_request_body may issue a sync httpx.get (up to 5s timeout) - # via _get_gcs_object_content_type to fetch GCS object metadata. Run the - # whole sync transformation on a worker thread so it does not block the - # async event loop. - return await asyncify(transform_request_body)( - messages=inlined_messages, - model=model, - custom_llm_provider=custom_llm_provider, - litellm_params=litellm_params, - cached_content=cached_content, - optional_params=optional_params, - ) - - return transform_request_body( + transform: Final = partial( + transform_request_body, messages=inlined_messages, model=model, custom_llm_provider=custom_llm_provider, @@ -1402,6 +1455,21 @@ async def async_transform_request_body( cached_content=cached_content, optional_params=optional_params, ) + # _transform_request_body may issue a sync httpx.get (up to 5s timeout) + # via _get_gcs_object_content_type to fetch GCS object metadata. Run the + # whole sync transformation on a worker thread so it does not block the + # async event loop. + request_body: Final[RequestBody] = ( + await asyncify(transform)() + if _openai_messages_may_need_sync_gcs_metadata_fetch(inlined_messages) + else transform() + ) + return _with_default_output_budget( + request_body, + model, + custom_llm_provider, + litellm_params.get("model_info"), # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # litellm_params is an untyped dict + ) def _default_user_message_when_system_message_passed() -> ChatCompletionUserMessage: diff --git a/tests/unit/llms/vertex_ai/gemini/test_long_decode.py b/tests/unit/llms/vertex_ai/gemini/test_long_decode.py index 4465ad7f3df..3fe9a5c7a92 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_long_decode.py +++ b/tests/unit/llms/vertex_ai/gemini/test_long_decode.py @@ -516,9 +516,16 @@ async def test_unary_answer_cut_at_the_callers_cap_is_returned_with_its_token_in assert response.choices[0].message.provider_specific_fields["continuation_token"] == "dG9rLTE=" +@pytest.mark.parametrize( + "choice_count_request", + [{"max_tokens": 100_000, "n": 2}, {"extra_body": {"generationConfig": {"candidateCount": 2}}}], +) @pytest.mark.parametrize("is_async", [True, False]) @pytest.mark.asyncio -async def test_unary_request_for_more_than_one_choice_is_never_continued(is_async: bool): +async def test_unary_request_for_more_than_one_choice_is_never_continued( + monkeypatch: pytest.MonkeyPatch, is_async: bool, choice_count_request: Mapping[str, object] +): + monkeypatch.setenv(BUDGET_ENV, "100000") sent: Final[list[dict[str, object]]] = [] # Vertex AI answered 400 "Candidate count greater than 1 is not supported when continuation_token is set" # on 2026-10-13 @@ -526,9 +533,8 @@ async def test_unary_request_for_more_than_one_choice_is_never_continued(is_asyn kwargs: Final = { "model": MODEL, "messages": list(MESSAGES), - "max_tokens": 100_000, - "n": 2, "api_key": "fake-gemini-key", + **choice_count_request, } response: Final = ( @@ -548,6 +554,42 @@ async def test_unary_request_for_more_than_one_choice_is_never_continued(is_asyn ] +@pytest.mark.parametrize( + "one_choice_request", + [ + {"max_tokens": 100_000, "n": 0}, + {"extra_body": {"generationConfig": {"candidateCount": None}}}, + {"extra_body": {"generationConfig": {"candidateCount": 0}}}, + ], +) +@pytest.mark.parametrize("is_async", [True, False]) +@pytest.mark.asyncio +async def test_unary_request_whose_candidate_count_google_reads_as_one_is_continued( + monkeypatch: pytest.MonkeyPatch, is_async: bool, one_choice_request: Mapping[str, object] +): + monkeypatch.setenv(BUDGET_ENV, "100000") + sent: Final[list[dict[str, object]]] = [] + hops: Final = (httpx.Response(200, json=HOP_1), httpx.Response(200, json=HOP_2)) + kwargs: Final = { + "model": MODEL, + "messages": list(MESSAGES), + "api_key": "fake-gemini-key", + **one_choice_request, + } + + response: Final = ( + await litellm.acompletion(**kwargs, client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay(hops, sent)))) + if is_async + else litellm.completion( + **kwargs, client=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_replay(hops, sent)))) + ) + ) + + assert sent[1] == {**sent[0], "continuationToken": "dG9rLTE="} + assert response.choices[0].message.content == "hello world" + assert response.choices[0].finish_reason == "stop" + + @pytest.mark.asyncio async def test_unary_failed_continuation_hop_returns_the_answer_so_far_with_its_token(): sent: Final[list[dict[str, object]]] = [] @@ -1295,6 +1337,468 @@ async def test_vertex_long_decode_keeps_the_callers_own_authorization_header_on_ assert seen == ["Bearer mine", "Bearer mine"] +BUDGET_ENV: Final = "LITELLM_GEMINI_DEFAULT_MAX_OUTPUT_TOKENS" +ARGON: Final = "gemini/gemini-4-argon" +UNMAPPED_GEMINI: Final = "gemini/gemini-99-unreleased" +OUTPUT_LIMIT_KEYS: Final = ("max_output_tokens", "maxOutputTokens") + + +def _output_limits(request: Mapping[str, object]) -> dict[str, object]: + generation_config: Final = request.get("generationConfig", {}) + assert isinstance(generation_config, dict) + return {key: value for key, value in generation_config.items() if key in OUTPUT_LIMIT_KEYS} + + +def _cost_map_max_output_tokens(model: str) -> int: + provider, name = model.split("/", 1) + max_output_tokens: Final = litellm.get_model_info(model=name, custom_llm_provider=provider)["max_output_tokens"] + assert isinstance(max_output_tokens, int) + return max_output_tokens + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("is_async", [True, False]) +@pytest.mark.asyncio +async def test_request_with_no_output_limit_keeps_continuing_until_the_env_budget_is_spent( + monkeypatch: pytest.MonkeyPatch, is_async: bool, stream: bool +): + monkeypatch.setenv(BUDGET_ENV, "50") + sent: Final[list[dict[str, object]]] = [] + hops: Final = ( + (_sse(STREAM_FIRST_CHUNK, STREAM_HOP_1_END), _sse(HOP_2_TRUNCATED), _sse(HOP_2)) + if stream + else ( + httpx.Response(200, json=HOP_1), + httpx.Response(200, json=HOP_2_TRUNCATED), + httpx.Response(200, json=HOP_2), + ) + ) + kwargs: Final = {"model": ARGON, "messages": list(MESSAGES), "stream": stream, "api_key": "fake-gemini-key"} + transport: Final = httpx.MockTransport(_replay(hops, sent)) + + response: Final = ( + await litellm.acompletion(**kwargs, client=AsyncHTTPHandler(transport=transport)) + if is_async + else litellm.completion(**kwargs, client=HTTPHandler(client=httpx.Client(transport=transport))) + ) + chunks: Final = ([chunk async for chunk in response] if is_async else list(response)) if stream else [] + + assert len(sent) == 2 + assert _output_limits(sent[0]) == {"max_output_tokens": 50} + if stream: + assert _stream_text(chunks) == "hello world" + assert _stream_finish_reasons(chunks) == ["length"] + assert {"continuation_token": "dG9rLTI="} in _delta_fields(chunks) + else: + assert response.choices[0].message.content == "hello world" + assert response.choices[0].finish_reason == "length" + assert response.choices[0].message.provider_specific_fields["continuation_token"] == "dG9rLTI=" + + +@pytest.mark.parametrize( + "budget, model, request_params, expected_limits", + [ + ("50", ARGON, {"max_tokens": 30}, {"max_output_tokens": 30}), + ("50", ARGON, {"max_completion_tokens": 30}, {"max_output_tokens": 30}), + ( + "50", + ARGON, + {"extra_body": {"generationConfig": {"maxOutputTokens": 30}}}, + {"maxOutputTokens": 30}, + ), + (None, ARGON, {}, {}), + ("", ARGON, {}, {}), + ("a lot", ARGON, {}, {}), + ("0", ARGON, {}, {}), + ("-5", ARGON, {}, {}), + ("50", UNMAPPED_GEMINI, {}, {}), + ], +) +@pytest.mark.asyncio +async def test_env_budget_applies_only_to_a_request_that_sets_no_output_limit_of_its_own( + monkeypatch: pytest.MonkeyPatch, + budget: str | None, + model: str, + request_params: Mapping[str, object], + expected_limits: Mapping[str, object], +): + if budget is None: + monkeypatch.delenv(BUDGET_ENV, raising=False) + else: + monkeypatch.setenv(BUDGET_ENV, budget) + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=model, + messages=list(MESSAGES), + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + **request_params, + ) + + assert _output_limits(sent[0]) == expected_limits + + +@pytest.mark.parametrize( + "modalities, expected_limits", + [ + (["audio"], {}), + (["text", "audio"], {}), + (["text", "image"], {}), + (["text"], {"max_output_tokens": 50}), + ], +) +@pytest.mark.asyncio +async def test_env_budget_applies_only_to_a_request_asking_for_text_output( + monkeypatch: pytest.MonkeyPatch, modalities: list[str], expected_limits: Mapping[str, object] +): + monkeypatch.setenv(BUDGET_ENV, "50") + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=ARGON, + messages=list(MESSAGES), + modalities=modalities, + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == expected_limits + + +def _ai_studio_model_in_mode(mode: str) -> str: + return next( + name + for name, row in litellm.model_cost.items() + if name.startswith("gemini/") and row.get("mode") == mode and row.get("max_output_tokens") + ) + + +@pytest.mark.parametrize("mode", ["audio_speech", "image_generation"]) +@pytest.mark.asyncio +async def test_env_budget_skips_a_model_the_cost_map_does_not_list_as_a_chat_model( + monkeypatch: pytest.MonkeyPatch, mode: str +): + monkeypatch.setenv(BUDGET_ENV, "50") + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=_ai_studio_model_in_mode(mode), + messages=list(MESSAGES), + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == {} + + +@pytest.mark.parametrize( + "output_modalities, expected_limits", + [ + ({}, {"max_output_tokens": 50}), + ({"supported_output_modalities": ["text"]}, {"max_output_tokens": 50}), + ({"supported_output_modalities": ["text", "audio"]}, {"max_output_tokens": 50}), + ({"supported_output_modalities": ["audio"]}, {}), + ({"supported_output_modalities": "audio"}, {}), + ], +) +@pytest.mark.asyncio +async def test_env_budget_applies_only_to_a_chat_model_the_cost_map_lists_as_answering_text( + monkeypatch: pytest.MonkeyPatch, output_modalities: Mapping[str, object], expected_limits: Mapping[str, object] +): + monkeypatch.setenv(BUDGET_ENV, "50") + litellm.register_model( + { + UNMAPPED_GEMINI: { + "litellm_provider": "gemini", + "mode": "chat", + "max_output_tokens": 1000, + **output_modalities, + } + } + ) + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=UNMAPPED_GEMINI, + messages=list(MESSAGES), + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == expected_limits + + +@pytest.mark.parametrize( + "extra_body", + [ + {"generation_config": {"temperature": 0.1}}, + {"generation_config": {"max_output_tokens": 30}}, + {"generationConfig": {"response_modalities": ["AUDIO"]}}, + ], +) +@pytest.mark.asyncio +async def test_env_budget_leaves_a_snake_case_generation_config_from_extra_body_alone( + monkeypatch: pytest.MonkeyPatch, extra_body: Mapping[str, object] +): + monkeypatch.setenv(BUDGET_ENV, "50") + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=ARGON, + messages=list(MESSAGES), + extra_body=dict(extra_body), + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == {} + + +def _gemini_chat_models_answering_no_text() -> tuple[str, ...]: + rows: Final = ( + (name, row) + for name, row in litellm.model_cost.items() + if row.get("mode") == "chat" and "text" not in row.get("supported_output_modalities", ("text",)) + ) + return tuple( + name if name.startswith("gemini/") else f"vertex_ai/{name}" + for name, row in rows + if row.get("litellm_provider") in ("gemini", "vertex_ai-language-models") + ) + + +@pytest.mark.asyncio +async def test_env_budget_skips_every_chat_model_the_cost_map_lists_as_answering_no_text( + monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint +): + monkeypatch.setenv(BUDGET_ENV, "50") + models: Final = _gemini_chat_models_answering_no_text() + sent: Final[list[dict[str, object]]] = [] + replies: Final = tuple(httpx.Response(200, json=HOP_2) for _ in models) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent))) + + for model in models: + await litellm.acompletion( + model=model, + messages=list(MESSAGES), + api_key="fake-gemini-key", + vertex_project="p", + vertex_location="us-central1", + vertex_credentials=token_endpoint.credentials, + client=client, + ) + + assert models + assert {model: _output_limits(request) for model, request in zip(models, sent, strict=True)} == dict.fromkeys( + models, {} + ) + + +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.asyncio +async def test_env_budget_leaves_a_continuation_token_resume_with_no_limit_alone( + monkeypatch: pytest.MonkeyPatch, stream: bool +): + monkeypatch.setenv(BUDGET_ENV, "50") + sent: Final[list[dict[str, object]]] = [] + reply: Final = _sse(HOP_2) if stream else httpx.Response(200, json=HOP_2) + + response: Final = await litellm.acompletion( + model=ARGON, + messages=list(MESSAGES), + continuation_token="dG9rLTE=", + stream=stream, + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((reply,), sent))), + ) + if stream: + _ = [chunk async for chunk in response] + + assert sent[0]["continuationToken"] == "dG9rLTE=" + assert _output_limits(sent[0]) == {} + + +@pytest.mark.parametrize("model", ["gemini/gemini-2.5-flash", "vertex_ai/gemini-2.5-flash", ARGON]) +@pytest.mark.asyncio +async def test_env_budget_above_a_models_output_limit_sends_that_limit_instead( + monkeypatch: pytest.MonkeyPatch, token_endpoint: _TokenEndpoint, model: str +): + budget: Final = 2 * _cost_map_max_output_tokens(model) + monkeypatch.setenv(BUDGET_ENV, str(budget)) + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=model, + messages=list(MESSAGES), + api_key="fake-gemini-key", + vertex_project="p", + vertex_location="us-central1", + vertex_credentials=token_endpoint.credentials, + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == {"max_output_tokens": _cost_map_max_output_tokens(model)} + + +@pytest.mark.asyncio +async def test_env_budget_never_goes_above_the_cost_map_limit_when_a_deployment_overrides_it( + monkeypatch: pytest.MonkeyPatch, +): + model: Final = "gemini/gemini-2.5-flash" + model_limit: Final = _cost_map_max_output_tokens(model) + monkeypatch.setenv(BUDGET_ENV, str(2 * model_limit)) + router: Final = litellm.Router( + model_list=[ + {"model_name": "plain", "litellm_params": {"model": model, "api_key": "fake-gemini-key"}}, + { + "model_name": "raised", + "litellm_params": {"model": model, "api_key": "fake-gemini-key"}, + "model_info": {"max_output_tokens": 4 * model_limit}, + }, + ] + ) + sent: Final[list[dict[str, object]]] = [] + replies: Final = (httpx.Response(200, json=HOP_2), httpx.Response(200, json=HOP_2)) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent))) + + for model_name in ("plain", "raised"): + await router.acompletion(model=model_name, messages=list(MESSAGES), client=client) + + sent_limits: Final = [_output_limits(request).get("max_output_tokens", 0) for request in sent] + assert len(sent_limits) == 2 + assert all(isinstance(limit, int) and limit <= model_limit for limit in sent_limits) + + +@pytest.mark.asyncio +async def test_env_budget_caps_each_deployment_at_its_own_model_info_limit_and_never_at_a_siblings( + monkeypatch: pytest.MonkeyPatch, +): + model: Final = "gemini/gemini-2.5-flash" + budget: Final = _cost_map_max_output_tokens(model) // 2 + low_limit: Final = budget // 4 + monkeypatch.setenv(BUDGET_ENV, str(budget)) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "low", + "litellm_params": {"model": model, "api_key": "fake-gemini-key"}, + "model_info": {"max_output_tokens": low_limit}, + }, + {"model_name": "plain", "litellm_params": {"model": model, "api_key": "fake-gemini-key"}}, + ] + ) + sent: Final[list[dict[str, object]]] = [] + replies: Final = (httpx.Response(200, json=HOP_2), httpx.Response(200, json=HOP_2)) + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(_replay(replies, sent))) + + for model_name in ("plain", "low"): + await router.acompletion(model=model_name, messages=list(MESSAGES), client=client) + + assert [_output_limits(request) for request in sent] == [ + {"max_output_tokens": budget}, + {"max_output_tokens": low_limit}, + ] + + +@pytest.mark.parametrize("deployment_limit", [0, -1]) +@pytest.mark.asyncio +async def test_env_budget_ignores_a_deployment_model_info_limit_that_is_not_positive( + monkeypatch: pytest.MonkeyPatch, deployment_limit: int +): + model: Final = "gemini/gemini-2.5-flash" + budget: Final = _cost_map_max_output_tokens(model) // 2 + monkeypatch.setenv(BUDGET_ENV, str(budget)) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "odd", + "litellm_params": {"model": model, "api_key": "fake-gemini-key"}, + "model_info": {"max_output_tokens": deployment_limit}, + } + ] + ) + sent: Final[list[dict[str, object]]] = [] + + await router.acompletion( + model="odd", + messages=list(MESSAGES), + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == {"max_output_tokens": budget} + + +@pytest.mark.asyncio +async def test_env_budget_on_a_direct_call_never_goes_above_the_cost_map_limit_after_a_router_raised_it( + monkeypatch: pytest.MonkeyPatch, +): + model: Final = "gemini/gemini-2.5-flash" + model_limit: Final = _cost_map_max_output_tokens(model) + monkeypatch.setenv(BUDGET_ENV, str(2 * model_limit)) + litellm.Router( + model_list=[ + { + "model_name": "raised", + "litellm_params": {"model": model, "api_key": "fake-gemini-key"}, + "model_info": {"max_output_tokens": 4 * model_limit}, + } + ] + ) + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=model, + messages=list(MESSAGES), + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == {"max_output_tokens": model_limit} + + +@pytest.mark.asyncio +async def test_env_budget_on_a_direct_call_follows_a_lower_limit_set_with_register_model( + monkeypatch: pytest.MonkeyPatch, +): + model: Final = "gemini/gemini-2.5-flash" + lowered_limit: Final = _cost_map_max_output_tokens(model) // 4 + monkeypatch.setenv(BUDGET_ENV, str(2 * lowered_limit)) + litellm.register_model({model: {"litellm_provider": "gemini", "max_output_tokens": lowered_limit}}) + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=model, + messages=list(MESSAGES), + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert _output_limits(sent[0]) == {"max_output_tokens": lowered_limit} + + +@pytest.mark.parametrize( + "generation_config", + [None, {"responseModalities": None}, {"responseModalities": {"TEXT": 1}}], +) +@pytest.mark.asyncio +async def test_env_budget_sends_a_generation_config_of_an_unexpected_shape_as_it_came( + monkeypatch: pytest.MonkeyPatch, generation_config: object +): + monkeypatch.setenv(BUDGET_ENV, "50") + sent: Final[list[dict[str, object]]] = [] + + await litellm.acompletion( + model=ARGON, + messages=list(MESSAGES), + extra_body={"generationConfig": generation_config}, + api_key="fake-gemini-key", + client=AsyncHTTPHandler(transport=httpx.MockTransport(_replay((httpx.Response(200, json=HOP_2),), sent))), + ) + + assert sent[0]["generationConfig"] == generation_config + + def test_initial_prompt_tokens_is_absent_on_a_single_hop_and_the_first_hops_count_otherwise(): assert initial_prompt_tokens_of((HOP_1,)) is None assert initial_prompt_tokens_of((HOP_1, HOP_2)) == 10 From 398579508ac1505e99902d95687c74442c54a5fc Mon Sep 17 00:00:00 2001 From: jesus-berri Date: Tue, 20 Oct 2026 19:52:29 -0700 Subject: [PATCH 045/235] feat(ui): add Delete to the guardrail detail page and share one delete flow with the table row (#45893) * feat(ui): add Delete to the guardrail detail Settings card for DB guardrails Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): hoist guardrail delete test fixtures to stay within the inline-object lint budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(ui): share one guardrail delete modal between the table row and the detail page The table-row delete now has the same dangling-reference warning and typed-name confirmation as the detail page. Tests now assert the success and error toasts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): warn that deleting a guardrail used in a policy pipeline step blocks requests with a 400 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(ui): cover hiding guardrail Delete while settings are being edited Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../_components/GuardrailDeleteModal.test.tsx | 78 ++++++++++ .../_components/GuardrailDeleteModal.tsx | 68 +++++++++ .../_components/GuardrailSettingsActions.tsx | 51 +++++++ .../GuardrailsPanel.integration.test.tsx | 135 ++++++++++++++++++ .../_components/GuardrailsPanel.test.tsx | 30 +++- .../_components/GuardrailsPanel.tsx | 69 ++------- .../guardrail_info.characterization.test.tsx | 3 +- .../_components/guardrail_info.test.tsx | 109 ++++++++++++-- .../guardrails/_components/guardrail_info.tsx | 28 ++-- 9 files changed, 489 insertions(+), 82 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailDeleteModal.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailDeleteModal.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailSettingsActions.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.integration.test.tsx diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailDeleteModal.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailDeleteModal.test.tsx new file mode 100644 index 00000000000..0d74b714ee6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailDeleteModal.test.tsx @@ -0,0 +1,78 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { GuardrailDeleteModal } from "./GuardrailDeleteModal"; +import { deleteGuardrailCall } from "@/components/networking"; +import { toast } from "@/lib/toast"; +import { fireEvent, renderWithProviders, screen, waitFor } from "@/../tests/test-utils"; + +vi.mock("@/components/networking", () => ({ deleteGuardrailCall: vi.fn() })); +vi.mock("@/lib/toast", () => ({ toast: { success: vi.fn(), fromError: vi.fn() } })); + +const GUARDRAIL = { + guardrail_id: "gr-1", + guardrail_name: "word-block", + litellm_params: { guardrail: "litellm_content_filter", mode: "pre_call", default_on: true }, +}; + +const renderModal = () => { + const onClose = vi.fn(); + const onDeleted = vi.fn(); + renderWithProviders( + , + ); + return { onClose, onDeleted }; +}; + +const confirmDelete = () => { + fireEvent.change(screen.getByPlaceholderText(GUARDRAIL.guardrail_name), { + target: { value: GUARDRAIL.guardrail_name }, + }); + fireEvent.click(screen.getByRole("button", { name: "Delete" })); +}; + +describe("GuardrailDeleteModal", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("warns about name references and only enables Delete once the exact name is typed", () => { + renderModal(); + + const warning = screen.getByText(/silently stop running it/); + expect(warning).toHaveTextContent("Keys, teams, and policies that list this guardrail"); + expect(warning).toHaveTextContent("Policy pipeline steps that use it will block every request with a 400 error"); + expect(warning).toHaveTextContent("unless the step's on_error is next or allow"); + const confirmButton = screen.getByRole("button", { name: "Delete" }); + expect(confirmButton).toBeDisabled(); + fireEvent.change(screen.getByPlaceholderText(GUARDRAIL.guardrail_name), { target: { value: "word" } }); + expect(confirmButton).toBeDisabled(); + fireEvent.change(screen.getByPlaceholderText(GUARDRAIL.guardrail_name), { + target: { value: GUARDRAIL.guardrail_name }, + }); + expect(confirmButton).toBeEnabled(); + }); + + it("shows a success toast naming the guardrail and calls onDeleted after a successful delete", async () => { + vi.mocked(deleteGuardrailCall).mockResolvedValue({}); + const { onClose, onDeleted } = renderModal(); + + confirmDelete(); + + await waitFor(() => expect(onDeleted).toHaveBeenCalledTimes(1)); + expect(deleteGuardrailCall).toHaveBeenCalledWith("tok", GUARDRAIL.guardrail_id); + expect(toast.success).toHaveBeenCalledWith('Guardrail "word-block" deleted successfully'); + expect(toast.fromError).not.toHaveBeenCalled(); + expect(onClose).toHaveBeenCalledTimes(1); + }); + + it("shows an error toast and does not call onDeleted when the delete fails", async () => { + vi.mocked(deleteGuardrailCall).mockRejectedValue(new Error("boom")); + const { onClose, onDeleted } = renderModal(); + + confirmDelete(); + + await waitFor(() => expect(onClose).toHaveBeenCalledTimes(1)); + expect(toast.fromError).toHaveBeenCalledWith("Failed to delete guardrail"); + expect(toast.success).not.toHaveBeenCalled(); + expect(onDeleted).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailDeleteModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailDeleteModal.tsx new file mode 100644 index 00000000000..a3ed15b630f --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailDeleteModal.tsx @@ -0,0 +1,68 @@ +import React, { useState } from "react"; +import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; +import { deleteGuardrailCall } from "@/components/networking"; +import { toast } from "@/lib/toast"; +import { formatGuardrailMode, getGuardrailLogoAndName } from "./guardrail_info_helpers"; + +const DANGLING_REFERENCE_WARNING = + "Keys, teams, and policies that list this guardrail will silently stop running it. Policy pipeline steps that use it will block every request with a 400 error, unless the step's on_error is next or allow (or its on_fail, when on_error is not set)."; + +export interface DeletableGuardrail { + guardrail_id: string; + guardrail_name?: string | null; + litellm_params?: { + guardrail?: string; + mode?: unknown; + default_on?: boolean | null; + } | null; +} + +interface GuardrailDeleteModalProps { + guardrail: DeletableGuardrail | null; + accessToken: string | null; + onClose: () => void; + onDeleted: () => void; +} + +export function GuardrailDeleteModal({ guardrail, accessToken, onClose, onDeleted }: GuardrailDeleteModalProps) { + const [isDeleting, setIsDeleting] = useState(false); + const guardrailName = guardrail ? guardrail.guardrail_name || guardrail.guardrail_id : ""; + const providerDisplayName = getGuardrailLogoAndName(guardrail?.litellm_params?.guardrail || "").displayName; + + const handleDelete = async () => { + if (!guardrail || !accessToken) return; + setIsDeleting(true); + try { + await deleteGuardrailCall(accessToken, guardrail.guardrail_id); + toast.success(`Guardrail "${guardrailName}" deleted successfully`); + onClose(); + onDeleted(); + } catch { + toast.fromError("Failed to delete guardrail"); + onClose(); + } finally { + setIsDeleting(false); + } + }; + + return ( + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailSettingsActions.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailSettingsActions.tsx new file mode 100644 index 00000000000..46bc09e7383 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailSettingsActions.tsx @@ -0,0 +1,51 @@ +import { Code, Trash2 } from "lucide-react"; +import React, { useState } from "react"; +import { Button } from "@/components/ui/button"; +import { DeletableGuardrail, GuardrailDeleteModal } from "./GuardrailDeleteModal"; + +interface GuardrailSettingsActionsProps { + guardrail: DeletableGuardrail & { guardrail_definition_location?: string }; + isEditing: boolean; + onEdit: () => void; + onEditCode: () => void; + accessToken: string | null; + onDeleted: () => void; +} + +export function GuardrailSettingsActions({ + guardrail, + isEditing, + onEdit, + onEditCode, + accessToken, + onDeleted, +}: GuardrailSettingsActionsProps) { + const [isModalOpen, setIsModalOpen] = useState(false); + + if (isEditing || guardrail.guardrail_definition_location === "config") return null; + + return ( + <> + {guardrail.litellm_params?.guardrail === "custom_code" ? ( + + ) : ( + + )} + + setIsModalOpen(false)} + onDeleted={onDeleted} + /> + + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.integration.test.tsx new file mode 100644 index 00000000000..19eb838da6b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.integration.test.tsx @@ -0,0 +1,135 @@ +import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; +import GuardrailsPanel from "./GuardrailsPanel"; +import { + deleteGuardrailCall, + getGuardrailInfo, + getGuardrailProviderSpecificParams, + getGuardrailsList, + getGuardrailUISettings, +} from "@/components/networking"; +import { fireEvent, renderWithProviders, screen, waitFor, within } from "@/../tests/test-utils"; + +vi.mock("@/components/networking", () => ({ + getGuardrailsList: vi.fn(), + deleteGuardrailCall: vi.fn(), + getGuardrailInfo: vi.fn(), + getGuardrailUISettings: vi.fn(), + getGuardrailProviderSpecificParams: vi.fn(), + updateGuardrailCall: vi.fn(), +})); + +vi.mock("./add_guardrail_form", () => ({ __esModule: true, default: () => null })); +vi.mock("./guardrail_garden", () => ({ __esModule: true, default: () => null })); +vi.mock("./GuardrailTestPlayground", () => ({ __esModule: true, default: () => null })); +vi.mock("./TeamGuardrailsTab", () => ({ TeamGuardrailsTab: () => null })); + +const DB_GUARDRAIL = { + guardrail_id: "db-guardrail-1", + guardrail_name: "db-word-block", + litellm_params: { guardrail: "litellm_content_filter", mode: "pre_call", default_on: true }, + guardrail_info: null, + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + guardrail_definition_location: "db", +}; + +const CONFIG_GUARDRAIL = { + ...DB_GUARDRAIL, + guardrail_id: "config-guardrail-1", + guardrail_name: "cfg-word-block", + guardrail_definition_location: "config", +}; + +beforeAll(() => { + Object.defineProperty(window, "matchMedia", { + writable: true, + value: vi.fn().mockImplementation((query: string) => ({ + matches: false, + media: query, + onchange: null, + addListener: vi.fn(), + removeListener: vi.fn(), + addEventListener: vi.fn(), + removeEventListener: vi.fn(), + dispatchEvent: vi.fn(), + })), + }); +}); + +const UI_SETTINGS = { + supported_entities: [], + supported_actions: [], + pii_entity_categories: [], + supported_modes: ["pre_call", "post_call"], +}; + +describe("GuardrailsPanel delete from the guardrail detail Settings card", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(getGuardrailsList).mockResolvedValue({ guardrails: [DB_GUARDRAIL, CONFIG_GUARDRAIL] }); + vi.mocked(getGuardrailInfo).mockImplementation(async (_token: string, id: string) => + id === CONFIG_GUARDRAIL.guardrail_id ? CONFIG_GUARDRAIL : DB_GUARDRAIL, + ); + vi.mocked(getGuardrailUISettings).mockResolvedValue(UI_SETTINGS); + vi.mocked(getGuardrailProviderSpecificParams).mockResolvedValue({}); + }); + + const renderDetail = (guardrailId: string) => + renderWithProviders(, { + searchParams: `?guardrail=${guardrailId}`, + }); + + it("deletes a DB guardrail only after the admin types its name, then returns to the refreshed table", async () => { + vi.mocked(deleteGuardrailCall).mockResolvedValue({}); + renderDetail(DB_GUARDRAIL.guardrail_id); + + fireEvent.click(await screen.findByRole("tab", { name: "Settings" })); + fireEvent.click(await screen.findByRole("button", { name: "Delete" })); + + const modal = within(await screen.findByRole("dialog")); + expect( + modal.getByText(/pipeline steps that use it will block every request with a 400 error/i), + ).toBeInTheDocument(); + const confirmButton = modal.getByRole("button", { name: "Delete" }); + expect(confirmButton).toBeDisabled(); + + const confirmationInput = modal.getByPlaceholderText(DB_GUARDRAIL.guardrail_name); + fireEvent.change(confirmationInput, { target: { value: "db-word" } }); + expect(confirmButton).toBeDisabled(); + fireEvent.change(confirmationInput, { target: { value: DB_GUARDRAIL.guardrail_name } }); + expect(confirmButton).toBeEnabled(); + + fireEvent.click(confirmButton); + + await waitFor(() => expect(deleteGuardrailCall).toHaveBeenCalledWith("test-token", DB_GUARDRAIL.guardrail_id)); + expect(await screen.findByRole("table")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Edit Settings" })).not.toBeInTheDocument(); + expect(getGuardrailsList).toHaveBeenCalledTimes(2); + }); + + it("keeps the admin on the detail page when the delete request fails", async () => { + vi.mocked(deleteGuardrailCall).mockRejectedValue(new Error("boom")); + renderDetail(DB_GUARDRAIL.guardrail_id); + + fireEvent.click(await screen.findByRole("tab", { name: "Settings" })); + fireEvent.click(await screen.findByRole("button", { name: "Delete" })); + const modal = within(await screen.findByRole("dialog")); + fireEvent.change(modal.getByPlaceholderText(DB_GUARDRAIL.guardrail_name), { + target: { value: DB_GUARDRAIL.guardrail_name }, + }); + fireEvent.click(modal.getByRole("button", { name: "Delete" })); + + await waitFor(() => expect(deleteGuardrailCall).toHaveBeenCalledTimes(1)); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + expect(screen.getByRole("button", { name: "Delete" })).toBeInTheDocument(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); + + it("does not offer Delete on a config guardrail's detail page", async () => { + renderDetail(CONFIG_GUARDRAIL.guardrail_id); + + fireEvent.click(await screen.findByRole("tab", { name: "Settings" })); + expect(await screen.findByRole("img", { name: "Config guardrail details" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Delete" })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.test.tsx index 7be120a4cb7..ac824f63be9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.test.tsx @@ -2,6 +2,7 @@ import { type UrlUpdateEvent } from "nuqs/adapters/testing"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import GuardrailsPanel from "./GuardrailsPanel"; import { getGuardrailsList, deleteGuardrailCall } from "@/components/networking"; +import { toast } from "@/lib/toast"; import { fireEvent, renderWithProviders, screen, waitFor, within } from "@/../tests/test-utils"; vi.mock("@/components/networking", () => ({ @@ -9,6 +10,8 @@ vi.mock("@/components/networking", () => ({ deleteGuardrailCall: vi.fn(), })); +vi.mock("@/lib/toast", () => ({ toast: { success: vi.fn(), fromError: vi.fn() } })); + vi.mock("./add_guardrail_form", () => ({ __esModule: true, default: () =>
Mock Add Guardrail Form
, @@ -140,13 +143,36 @@ describe("GuardrailsPanel", () => { expect(modal.getByText("Delete Guardrail")).toBeInTheDocument(); expect(modal.getByText("test-guardrail-1")).toBeInTheDocument(); expect(modal.getByText("Test Provider")).toBeInTheDocument(); + expect( + modal.getByText(/pipeline steps that use it will block every request with a 400 error/i), + ).toBeInTheDocument(); - fireEvent.click(modal.getByRole("button", { name: "Delete" })); + const confirmButton = modal.getByRole("button", { name: "Delete" }); + expect(confirmButton).toBeDisabled(); + fireEvent.change(modal.getByPlaceholderText("Test Guardrail"), { target: { value: "Test Guardrail" } }); + fireEvent.click(confirmButton); await waitFor(() => { expect(mockDeleteGuardrailCall).toHaveBeenCalledWith("test-token", "test-guardrail-1"); }); - expect(mockGetGuardrailsList).toHaveBeenCalledTimes(2); + await waitFor(() => expect(mockGetGuardrailsList).toHaveBeenCalledTimes(2)); + expect(toast.success).toHaveBeenCalledWith('Guardrail "Test Guardrail" deleted successfully'); + }); + + it("should show an error toast and not refetch when the row delete fails", async () => { + mockDeleteGuardrailCall.mockRejectedValue(new Error("boom")); + renderWithProviders(); + fireEvent.click(screen.getByText("Guardrails")); + + fireEvent.click(await screen.findByTestId("delete-button")); + const modal = within(await screen.findByRole("dialog")); + fireEvent.change(modal.getByPlaceholderText("Test Guardrail"), { target: { value: "Test Guardrail" } }); + fireEvent.click(modal.getByRole("button", { name: "Delete" })); + + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + expect(toast.fromError).toHaveBeenCalledWith("Failed to delete guardrail"); + expect(toast.success).not.toHaveBeenCalled(); + expect(mockGetGuardrailsList).toHaveBeenCalledTimes(1); }); it("should mount every tab panel up front so panel state survives tab switches", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx index 901e39004f3..b53d016e950 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailsPanel.tsx @@ -2,7 +2,7 @@ import { parseAsString, useQueryState } from "nuqs"; import React, { useState, useEffect } from "react"; import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"; import { ChevronDown, Code, Plus } from "lucide-react"; -import { getGuardrailsList, deleteGuardrailCall } from "@/components/networking"; +import { getGuardrailsList } from "@/components/networking"; import { buttonVariants } from "@/components/ui/button"; import { DropdownMenu, @@ -16,10 +16,8 @@ import GuardrailTable from "./guardrail_table"; import { isAdminRole } from "@/utils/roles"; import GuardrailInfoView from "./guardrail_info"; import GuardrailTestPlayground from "./GuardrailTestPlayground"; -import { toast } from "@/lib/toast"; import { Guardrail } from "@/components/guardrails/types"; -import DeleteResourceModal from "@/components/common_components/DeleteResourceModal"; -import { formatGuardrailMode, getGuardrailLogoAndName } from "./guardrail_info_helpers"; +import { GuardrailDeleteModal } from "./GuardrailDeleteModal"; import { CustomCodeModal } from "./custom_code"; import GuardrailGarden from "./guardrail_garden"; import { TeamGuardrailsTab } from "./TeamGuardrailsTab"; @@ -38,9 +36,7 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole const [isAddModalVisible, setIsAddModalVisible] = useState(false); const [isCustomCodeModalVisible, setIsCustomCodeModalVisible] = useState(false); const [isLoading, setIsLoading] = useState(false); - const [isDeleting, setIsDeleting] = useState(false); const [guardrailToDelete, setGuardrailToDelete] = useState(null); - const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const [selectedGuardrailId, setSelectedGuardrailId] = useQueryState( "guardrail", parseAsString.withOptions({ history: "push" }), @@ -97,40 +93,10 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole fetchGuardrails(); }; - const handleDeleteClick = (guardrailId: string, guardrailName: string) => { - const guardrail = guardrailsList.find((g) => g.guardrail_id === guardrailId) || null; - setGuardrailToDelete(guardrail); - setIsDeleteModalOpen(true); + const handleDeleteClick = (guardrailId: string) => { + setGuardrailToDelete(guardrailsList.find((g) => g.guardrail_id === guardrailId) || null); }; - const handleDeleteConfirm = async () => { - if (!guardrailToDelete || !accessToken) return; - - setIsDeleting(true); - try { - await deleteGuardrailCall(accessToken, guardrailToDelete.guardrail_id); - toast.success(`Guardrail "${guardrailToDelete.guardrail_name}" deleted successfully`); - await fetchGuardrails(); - } catch (error) { - console.error("Error deleting guardrail:", error); - toast.fromError("Failed to delete guardrail"); - } finally { - setIsDeleting(false); - setIsDeleteModalOpen(false); - setGuardrailToDelete(null); - } - }; - - const handleDeleteCancel = () => { - setIsDeleteModalOpen(false); - setGuardrailToDelete(null); - }; - - const providerDisplayName = - guardrailToDelete && guardrailToDelete.litellm_params - ? getGuardrailLogoAndName(guardrailToDelete.litellm_params.guardrail).displayName - : undefined; - return (
@@ -184,6 +150,10 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole { + closeGuardrailDetail(); + fetchGuardrails(); + }} accessToken={accessToken} isAdmin={isAdmin} /> @@ -210,24 +180,11 @@ const GuardrailsPanel: React.FC = ({ accessToken, userRole onSuccess={handleSuccess} /> - setGuardrailToDelete(null)} + onDeleted={fetchGuardrails} /> diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx index 1689cf93289..732c50647b1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.characterization.test.tsx @@ -84,7 +84,8 @@ const saveChanges = async (user: ReturnType) => { await user.click(screen.getByText("Save Changes")); }; -const renderView = () => render( {}} accessToken="123" isAdmin />); +const renderView = () => + render( {}} onDeleted={() => {}} accessToken="123" isAdmin />); const lastPayload = () => vi.mocked(networking.updateGuardrailCall).mock.calls.at(-1)?.[2]; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx index 836921104ff..ae057689b37 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.test.tsx @@ -10,6 +10,7 @@ vi.mock("@/components/networking", () => ({ getGuardrailUISettings: vi.fn(), getGuardrailProviderSpecificParams: vi.fn(), updateGuardrailCall: vi.fn(), + deleteGuardrailCall: vi.fn(), })); // Mock ContentFilterManager @@ -65,7 +66,9 @@ describe("Guardrail Info", () => { vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); - render( {}} accessToken="123" isAdmin={true} />); + render( + {}} onDeleted={() => {}} accessToken="123" isAdmin={true} />, + ); // Wait for the loading to complete and data to be rendered await waitFor(() => { @@ -103,7 +106,9 @@ describe("Guardrail Info", () => { vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); - render( {}} accessToken="123" isAdmin={true} />); + render( + {}} onDeleted={() => {}} accessToken="123" isAdmin={true} />, + ); expect(await screen.findAllByText("pre_call, post_call (tag-based)")).not.toHaveLength(0); }); @@ -131,7 +136,9 @@ describe("Guardrail Info", () => { vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); - render( {}} accessToken="123" isAdmin={true} />); + render( + {}} onDeleted={() => {}} accessToken="123" isAdmin={true} />, + ); const logo = await screen.findByAltText("Presidio PII logo"); expect(logo).toHaveAttribute("src", expect.stringContaining("microsoft_azure.svg")); @@ -162,7 +169,7 @@ describe("Guardrail Info", () => { vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); const { container } = render( - {}} accessToken="123" isAdmin={true} />, + {}} onDeleted={() => {}} accessToken="123" isAdmin={true} />, ); await waitFor(() => { @@ -212,7 +219,9 @@ describe("Guardrail Info", () => { vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); - render( {}} accessToken="123" isAdmin={true} />); + render( + {}} onDeleted={() => {}} accessToken="123" isAdmin={true} />, + ); await waitFor(() => { expect(screen.getByText("PII Entity Configuration")).toBeInTheDocument(); @@ -245,7 +254,9 @@ describe("Guardrail Info", () => { vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); vi.mocked(networking.updateGuardrailCall).mockResolvedValue({ status: "success" }); - render( {}} accessToken="123" isAdmin={true} />); + render( + {}} onDeleted={() => {}} accessToken="123" isAdmin={true} />, + ); await waitFor(() => { expect(screen.getByText("Settings")).toBeInTheDocument(); @@ -331,7 +342,9 @@ describe("Guardrail Info", () => { }); vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); - render( {}} accessToken="123" isAdmin={true} />); + render( + {}} onDeleted={() => {}} accessToken="123" isAdmin={true} />, + ); expect(await screen.findByRole("tab", { name: "Overview" })).toHaveAttribute("aria-selected", "true"); expect(screen.getByRole("tab", { name: "Settings" })).toHaveAttribute("aria-selected", "false"); @@ -355,10 +368,90 @@ describe("Guardrail Info when the guardrail cannot be loaded", () => { vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); const onClose = vi.fn(); - render(); + render( + {}} + accessToken="123" + isAdmin={true} + />, + ); expect(await screen.findByText("Guardrail not found")).toBeInTheDocument(); fireEvent.click(screen.getByRole("button", { name: /back to guardrails/i })); expect(onClose).toHaveBeenCalledTimes(1); }); + + describe("Delete from the Settings card", () => { + const dbGuardrail = { + guardrail_id: "123", + guardrail_name: "Test Guardrail", + litellm_params: { guardrail: "presidio", mode: "pre_call", default_on: true }, + guardrail_definition_location: "db", + }; + const uiSettings = { + supported_entities: [], + supported_actions: [], + pii_entity_categories: [], + supported_modes: ["pre_call"], + }; + const mockDbGuardrail = () => { + vi.mocked(networking.getGuardrailInfo).mockResolvedValue(dbGuardrail); + vi.mocked(networking.getGuardrailUISettings).mockResolvedValue(uiSettings); + vi.mocked(networking.getGuardrailProviderSpecificParams).mockResolvedValue({}); + }; + + const openDeleteModalAndConfirm = async () => { + fireEvent.click(await screen.findByRole("tab", { name: "Settings" })); + fireEvent.click(await screen.findByRole("button", { name: "Delete" })); + const modal = within(await screen.findByRole("dialog")); + expect(modal.getByText(/silently stop running it/)).toBeInTheDocument(); + const confirmButton = modal.getByRole("button", { name: "Delete" }); + expect(confirmButton).toBeDisabled(); + fireEvent.change(modal.getByPlaceholderText("Test Guardrail"), { target: { value: "Test Guardrail" } }); + fireEvent.click(confirmButton); + }; + + it("deletes the guardrail and calls onDeleted once the admin types its name", async () => { + mockDbGuardrail(); + vi.mocked(networking.deleteGuardrailCall).mockResolvedValue({}); + const onDeleted = vi.fn(); + render( + {}} onDeleted={onDeleted} accessToken="tok" isAdmin />, + ); + + await openDeleteModalAndConfirm(); + + await waitFor(() => expect(onDeleted).toHaveBeenCalledTimes(1)); + expect(networking.deleteGuardrailCall).toHaveBeenCalledWith("tok", "123"); + }); + + it("hides Delete while the settings are being edited", async () => { + mockDbGuardrail(); + render( {}} onDeleted={() => {}} accessToken="tok" isAdmin />); + + fireEvent.click(await screen.findByRole("tab", { name: "Settings" })); + fireEvent.click(await screen.findByRole("button", { name: "Edit Settings" })); + + expect(await screen.findByRole("button", { name: "Save Changes" })).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Delete" })).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Edit Settings" })).not.toBeInTheDocument(); + }); + + it("does not call onDeleted when the delete request fails", async () => { + mockDbGuardrail(); + vi.mocked(networking.deleteGuardrailCall).mockRejectedValue(new Error("boom")); + const onDeleted = vi.fn(); + render( + {}} onDeleted={onDeleted} accessToken="tok" isAdmin />, + ); + + await openDeleteModalAndConfirm(); + + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + expect(networking.deleteGuardrailCall).toHaveBeenCalledTimes(1); + expect(onDeleted).not.toHaveBeenCalled(); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index 2cfe2b79886..eacacf5e30e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -33,6 +33,7 @@ import { } from "./GuardrailFormField"; import ContentFilterManager, { formatContentFilterDataForAPI } from "./content_filter/ContentFilterManager"; import CustomCodeModal, { EditGuardrailData } from "./custom_code/CustomCodeModal"; +import { GuardrailSettingsActions } from "./GuardrailSettingsActions"; import { GuardrailModeCard } from "./GuardrailModeDisplay"; import { GuardrailReadOnlyDetails } from "./GuardrailReadOnlyDetails"; import { @@ -72,11 +73,12 @@ const SectionHeading: React.FC<{ children: React.ReactNode }> = ({ children }) = export interface GuardrailInfoProps { guardrailId: string; onClose: () => void; + onDeleted: () => void; accessToken: string | null; isAdmin: boolean; } -const GuardrailInfoView: React.FC = ({ guardrailId, onClose, accessToken, isAdmin }) => { +const GuardrailInfoView: React.FC = ({ guardrailId, onClose, onDeleted, accessToken, isAdmin }) => { const [guardrailData, setGuardrailData] = useState(null); const [guardrailProviderSpecificParams, setGuardrailProviderSpecificParams] = useState(null); const [loading, setLoading] = useState(true); @@ -686,25 +688,21 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, {isAdmin && ( -
-

Guardrail Settings

+
+

Guardrail Settings

{isConfigGuardrail && ( )} - {!isEditing && - !isConfigGuardrail && - (guardrailData.litellm_params?.guardrail === "custom_code" ? ( - - ) : ( - - ))} + setIsEditing(true)} + onEditCode={() => setCustomCodeModalVisible(true)} + accessToken={accessToken} + onDeleted={onDeleted} + />
{isEditing ? ( From 647ebd63e0d02349575991bc8f5178b9d48066e9 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 19:54:46 -0700 Subject: [PATCH 046/235] test(perf): emit per-location seconds, offending locations and the ten longest holds as event_loop_stalls breakdowns (#47757) Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/perf/setup/event_loop.py | 73 ++++++++++++++++--- .../e2e_harness/perf/setup/test_event_loop.py | 67 ++++++++++++++--- 2 files changed, 120 insertions(+), 20 deletions(-) diff --git a/tests/e2e/perf/setup/event_loop.py b/tests/e2e/perf/setup/event_loop.py index 1c582e24fd8..58144424544 100644 --- a/tests/e2e/perf/setup/event_loop.py +++ b/tests/e2e/perf/setup/event_loop.py @@ -18,7 +18,7 @@ import json import os import re from collections.abc import Iterable -from dataclasses import dataclass, replace +from dataclasses import asdict, dataclass, replace from datetime import datetime from itertools import groupby from pathlib import Path @@ -80,11 +80,24 @@ class StallRow: total_seconds: float max_seconds: float allowed_by: KnownStall | None + """Every hold in the row, longest first.""" + holds: tuple[float, ...] = () @property def offending(self) -> bool: return self.allowed_by is None or self.count > self.allowed_by.max_count + @property + def over_allowance(self) -> int: + return self.count if self.allowed_by is None else max(0, self.count - self.allowed_by.max_count) + + @property + def note(self) -> str: + allowed: Final = ( + f"{self.count} of {self.allowed_by.max_count} allowed" if self.allowed_by else "not on the allowlist" + ) + return f"{allowed}, max {self.max_seconds:.2f} s" + def _short(path: str) -> str: if "site-packages/" in path: @@ -161,6 +174,7 @@ def summarize(spec: EventLoopStalls, stalls: Iterable[Stall]) -> tuple[StallRow, total_seconds=sum(stall.seconds for stall in group), max_seconds=max(stall.seconds for stall in group), allowed_by=_allowed_by(spec, coroutine, where), + holds=tuple(sorted((stall.seconds for stall in group), reverse=True)), ) for (coroutine, where), group in ( (key, tuple(items)) for key, items in groupby(over, key=lambda s: (s.coroutine, s.where)) @@ -182,28 +196,65 @@ def offenders(rows: Iterable[StallRow]) -> tuple[str, ...]: ) -def _over_allowance(row: StallRow) -> int: - return row.count if row.allowed_by is None else max(0, row.count - row.allowed_by.max_count) +TOP_HOLDS: Final = 10 + + +@dataclass(frozen=True, slots=True) +class Slice: + """One `_breakdown` row in the shape the other metrics use (label, value, share) plus the allowlist verdict the + dashboard shows under the label and colours by. `share` is None where the rows are not parts of a total.""" + + label: str + value: float + share: float | None + note: str | None + over: bool + + +def _label(row: StallRow) -> str: + return f"{row.coroutine} {row.where}".strip() + + +def _slice(row: StallRow, value: float, total: float) -> Slice: + return Slice(label=_label(row), value=value, share=value / total, note=row.note, over=row.offending) + + +def _breakdown(name: str, slices: Iterable[Slice]) -> tuple[tuple[str, str], ...]: + rows: Final = tuple(asdict(row) for row in slices) + return ((f"perf_event_loop_{name}_breakdown", json.dumps(rows)),) if rows else () + + +def _longest_holds(table: tuple[StallRow, ...]) -> tuple[Slice, ...]: + holds: Final = sorted(((seconds, row) for row in table for seconds in row.holds), key=lambda hold: -hold[0]) + return tuple( + Slice(label=_label(row), value=seconds, share=None, note=None, over=row.offending) + for seconds, row in holds[:TOP_HOLDS] + ) def junit_properties(spec: EventLoopStalls, rows: Iterable[StallRow]) -> tuple[tuple[str, float | str], ...]: """The stall table as `perf_event_loop_*` properties for the dashboard: every hold over the threshold with the allowlist's total as its limit, the holds past their allowance (limit 0, the case's one assertion), the longest - and the summed hold, and a `_breakdown` of holds per location in the shape the other metrics use.""" + and the summed hold, each with a `_breakdown` the dashboard opens on click: holds and seconds per location, the + locations past their cap by how far over they are, and the ten longest single holds with where they landed.""" table: Final = tuple(rows) holds: Final = sum(row.count for row in table) - breakdown: Final = tuple( - {"label": f"{row.coroutine} {row.where}".strip(), "value": row.count, "share": row.count / holds} - for row in table - ) + offending: Final = sum(row.over_allowance for row in table) + total_seconds: Final = sum(row.total_seconds for row in table) return ( ("perf_event_loop_holds", holds), ("perf_event_loop_holds_limit", sum(known.max_count for known in spec.allowlist)), - ("perf_event_loop_offending_holds", sum(map(_over_allowance, table))), + ("perf_event_loop_offending_holds", offending), ("perf_event_loop_offending_holds_limit", 0), ("perf_event_loop_hold_seconds_max", max((row.max_seconds for row in table), default=0.0)), - ("perf_event_loop_hold_seconds_total", sum(row.total_seconds for row in table)), - *((("perf_event_loop_holds_breakdown", json.dumps(breakdown)),) if breakdown else ()), + ("perf_event_loop_hold_seconds_total", total_seconds), + *_breakdown("holds", (_slice(row, row.count, holds) for row in table)), + *_breakdown( + "offending_holds", + (_slice(row, row.over_allowance, offending) for row in table if row.over_allowance), + ), + *_breakdown("hold_seconds_total", (_slice(row, row.total_seconds, total_seconds) for row in table)), + *_breakdown("hold_seconds_max", _longest_holds(table)), ) diff --git a/tests/e2e_harness/perf/setup/test_event_loop.py b/tests/e2e_harness/perf/setup/test_event_loop.py index e03e985709b..75989b38275 100644 --- a/tests/e2e_harness/perf/setup/test_event_loop.py +++ b/tests/e2e_harness/perf/setup/test_event_loop.py @@ -37,8 +37,14 @@ class _Slice(BaseModel): model_config = ConfigDict(frozen=True) label: str - value: int - share: float + value: float + share: float | None + note: str | None + over: bool + + +def _breakdown(properties: dict[str, float | str], name: str) -> list[_Slice]: + return TypeAdapter(list[_Slice]).validate_json(str(properties[f"perf_event_loop_{name}_breakdown"])) def test_parse_keeps_the_coroutine_its_location_and_the_hold() -> None: @@ -116,13 +122,56 @@ def test_junit_properties_count_every_hold_and_only_the_holds_past_their_allowan assert properties["perf_event_loop_offending_holds_limit"] == 0 assert properties["perf_event_loop_hold_seconds_max"] == 0.731 assert math.isclose(float(properties["perf_event_loop_hold_seconds_total"]), 0.302 + 2 * 0.412 + 3 * 0.731) - breakdown: Final = TypeAdapter(list[_Slice]).validate_json(str(properties["perf_event_loop_holds_breakdown"])) - assert [(slice.label, slice.value) for slice in breakdown] == [ - ("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 3), - ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2), - ("LifespanOn.main() uvicorn/lifespan/on.py:86", 1), + breakdown: Final = _breakdown(properties, "holds") + assert [(slice.label, slice.value, slice.note, slice.over) for slice in breakdown] == [ + ( + "RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", + 3, + "3 of 2 allowed, max 0.73 s", + True, + ), + ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, "not on the allowlist, max 0.41 s", True), + ("LifespanOn.main() uvicorn/lifespan/on.py:86", 1, "1 of 3 allowed, max 0.30 s", False), ] - assert math.isclose(sum(slice.share for slice in breakdown), 1.0) + assert math.isclose(sum(slice.share or 0.0 for slice in breakdown), 1.0) + + +def test_the_offending_breakdown_lists_only_the_locations_past_their_cap_by_how_far_over_they_are() -> None: + rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, _STREAM, _HANDLER, _HANDLER, _HANDLER))) + offending: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "offending_holds") + assert [(slice.label, slice.value, slice.share, slice.over) for slice in offending] == [ + ("RequestResponseCycle.run_asgi() uvicorn/protocols/http/h11_impl.py:416", 1, 1 / 3, True), + ("TaskHandle._run_coro() anyio/_core/_tasks.py:275", 2, 2 / 3, True), + ] + clean: Final = dict(junit_properties(_SPEC, summarize(_SPEC, parse_stalls((_HANDLER, _HANDLER))))) + assert clean["perf_event_loop_offending_holds"] == 0 + assert "perf_event_loop_offending_holds_breakdown" not in clean + + +def test_the_seconds_breakdowns_carry_the_time_per_location_and_the_longest_single_holds() -> None: + long_handler: Final = _HANDLER.replace("0.731", "0.950") + rows: Final = summarize(_SPEC, parse_stalls((_STARTUP, _STREAM, long_handler, _HANDLER))) + properties: Final = dict(junit_properties(_SPEC, rows)) + per_location: Final = _breakdown(properties, "hold_seconds_total") + assert [(slice.label.split()[0], slice.value) for slice in per_location] == [ + ("RequestResponseCycle.run_asgi()", 0.950 + 0.731), + ("TaskHandle._run_coro()", 0.412), + ("LifespanOn.main()", 0.302), + ] + assert math.isclose(sum(slice.share or 0.0 for slice in per_location), 1.0) + longest: Final = _breakdown(properties, "hold_seconds_max") + assert [(slice.label.split()[0], slice.value, slice.share, slice.over) for slice in longest] == [ + ("RequestResponseCycle.run_asgi()", 0.950, None, False), + ("RequestResponseCycle.run_asgi()", 0.731, None, False), + ("TaskHandle._run_coro()", 0.412, None, True), + ("LifespanOn.main()", 0.302, None, False), + ] + + +def test_the_longest_holds_breakdown_stops_at_ten() -> None: + rows: Final = summarize(_SPEC, parse_stalls(tuple(_STARTUP for _ in range(12)))) + longest: Final = _breakdown(dict(junit_properties(_SPEC, rows)), "hold_seconds_max") + assert len(longest) == 10 def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None: @@ -130,4 +179,4 @@ def test_junit_properties_without_holds_report_zeros_and_no_breakdown() -> None: assert properties["perf_event_loop_holds"] == 0 assert properties["perf_event_loop_offending_holds"] == 0 assert properties["perf_event_loop_hold_seconds_max"] == 0 - assert "perf_event_loop_holds_breakdown" not in properties + assert not [name for name in properties if name.endswith("_breakdown")] From 0e24bbb8a2c4e250cde60ca92df92ea2ffe3de60 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 02:56:38 +0000 Subject: [PATCH 047/235] fix(responses): bill every round of a streamed MCP auto-execute on one spend row (#47445) * test(responses): cover billing every round of a streamed MCP auto-execute * fix(responses): bill every round of a streamed MCP auto-execute on one spend row * fix(responses): bill every round of a streamed MCP auto-execute on bridged providers A streamed /v1/responses MCP auto-execute on a provider bridged through chat completions (for example Anthropic) billed only round 1, because each round's chat stream dispatched on the shared logging object and the first dispatch won. Bridged rounds now go through the same one-row billing as native rounds. A round is billed as soon as its chat stream ends, before its closing events reach the client. The row carries the final response, usage and cost summed over every round, and the fixed margin once. A round served from the response cache is billed at zero. A follow-up chat stream that breaks bills the paid rounds on its failure row. A single-round stream logs exactly as before. * fix(responses): charge one fixed margin and keep bridged usage fields on streamed MCP rounds A follow-up round that fails after earlier rounds were billed now charges the per-request fixed margin once, and its failure row carries no cost breakdown that would disagree with the summed cost. A bridged multi-round stream logs a plain cache_hit flag like a single-round stream does, and its usage keeps the summed provider usage fields of every round. A new test covers a stream dropped after its event loop closed, which must release its parked log without an unawaited coroutine warning. * fix(responses): price each streamed MCP round on its own and bill every round when a guardrail blocks Each round of a streamed MCP auto-execute is now priced from its own uncached cost, so a provider that states its cost (xAI) charges the stated cost of every round plus the fixed margin once, and the cost breakdown always agrees with the billed cost. Summed usage goes through a JSON-safe dump, so a nested provider field such as xAI's context_details no longer breaks a failed stream's response.failed event. A post-call guardrail that blocks a multi-round bridged stream now drops the parked success log and bills every round on the failure row, and a failed follow-up round's failure row carries the summed breakdown of both rounds. * fix(responses): keep earlier MCP rounds on their success row when a later round fails When a later round of a streamed MCP auto-execute fails, the rounds before it are now logged on their success row, and the failure row bills only the failed round with no second fixed margin. That failure log runs on its own copy of the request's logging state, so its usage and cost never leak into the earlier rounds' row. A post-call guardrail that blocks a native multi-round stream keeps the summed success row beside the $0 failure row, as at the merge base, while a bridged stream still bills every round on its failure row. * fix(responses): keep a failed later MCP round on its own failure row A streamed MCP auto-execute whose later round failed wrote only the earlier rounds' success row. The failure was logged on a copy of the logging state, while the proxy builds its failure row from the request's own state, which by then held the success payload, so the failure row took the success row's id and was dropped. Now the failure is logged on the request's own state as at the merge base, and once a later round starts, the success row for the rounds before it is logged from a copy of the state taken at that moment, so the two rows never mix. * fix(proxy): let an MCP tool call reuse its request's max_parallel_requests slot * test(proxy): create the other-request task in a way Python 3.10 supports and mark its bindings Final * fix(proxy): keep a reused max_parallel slot until the request's last round An MCP tool call that reuses its request's max_parallel_requests slot no longer loses it to the first round's late success callback, so the key stays counted for the rounds still to run. The reused call also stops replacing the request's rate limit headers. * fix(responses): keep an unpriced streamed MCP round unpriced instead of billing it as free * fix(responses): log a bridged MCP stream as a chat completion with every round's usage --- litellm/litellm_core_utils/litellm_logging.py | 48 +- .../litellm_core_utils/streaming_handler.py | 20 +- litellm/proxy/common_request_processing.py | 3 + .../hooks/parallel_request_limiter_v3.py | 67 +- litellm/proxy/utils.py | 18 +- .../streaming_iterator.py | 14 +- .../responses/mcp/mcp_streaming_iterator.py | 104 +- litellm/responses/mcp/round_billing.py | 23 +- litellm/responses/mcp/stream_round_billing.py | 376 +++++++ litellm/responses/streaming_iterator.py | 18 +- .../test_litellm_logging.py | 21 + .../hooks/test_parallel_request_limiter_v3.py | 171 ++++ .../mcp/test_aresponses_api_with_mcp.py | 948 +++++++++++++++++- .../test_streaming_iterator_error_events.py | 81 +- 14 files changed, 1848 insertions(+), 64 deletions(-) create mode 100644 litellm/responses/mcp/stream_round_billing.py diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index eb2e7b8c240..a59672b5cb2 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -12,7 +12,7 @@ import subprocess import sys import time import traceback -from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence +from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from datetime import datetime as dt_object from functools import lru_cache @@ -575,6 +575,27 @@ class UncachedPrice: cost_breakdown: CostBreakdown | None +@dataclass(frozen=True, slots=True) +class BilledRoundsSpend: + usage: Usage + cost: float + cost_breakdown: CostBreakdown | None + + +@dataclass(frozen=True, slots=True) +class BilledRoundsDispatch: + """A parked success log for a stream bridged through chat completions that billed several rounds, and the spend + its failure log bills when a post-call hook rejects the response, as it does for a rejected one-round bridged + stream.""" + + dispatch: Coroutine[object, object, None] + spend: BilledRoundsSpend + + +def _fixed_margin(cost_breakdown: CostBreakdown | None) -> float: + return 0.0 if cost_breakdown is None else cost_breakdown.get("margin_fixed_amount", 0.0) + + def _timestamp_seconds(moment: object) -> float | None: if isinstance(moment, datetime.datetime): return moment.timestamp() @@ -2421,6 +2442,31 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["combined_usage_object"] = usage self.model_call_details["response_cost"] = response_cost + def record_failure_after_earlier_rounds( + self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float + ) -> None: + """Bill a failed round on the failure log after the rounds of its request before it were billed on the success + log. + + The fixed margin is a per-request fee, so it is charged once: the failed round's margin is dropped when the + earlier rounds already paid one.""" + from litellm.responses.mcp.round_billing import breakdown_less_repeated_fixed_margin + + failed_breakdown: Final = None if self.cost_breakdown is earlier.cost_breakdown else self.cost_breakdown + repeated_fixed_margin: Final = ( + 0.0 + if failed_breakdown is None + else min(_fixed_margin(earlier.cost_breakdown), _fixed_margin(failed_breakdown), earlier.cost, failed_cost) + ) + self.model_call_details["combined_usage_object"] = failed_usage + self.model_call_details["response_cost"] = failed_cost - repeated_fixed_margin + self.cost_breakdown = breakdown_less_repeated_fixed_margin(failed_breakdown, repeated_fixed_margin) + + def record_billed_rounds_for_failure(self, spend: BilledRoundsSpend) -> None: + """Bill every round of a fully streamed request on the failure log when a post-call hook rejects it.""" + self.record_partial_usage_for_failure(spend.usage, spend.cost) + self.cost_breakdown = spend.cost_breakdown + def attempt_scoped_copy(self) -> "Logging": """A copy with its own model_call_details, so logging one attempt of a request through it leaves the state the request's other attempts share untouched.""" diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index b210085c5cc..5e43157a55d 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -10,7 +10,7 @@ import traceback from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final, NoReturn, Protocol, TypeVar, cast +from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast import anyio import httpx @@ -60,6 +60,9 @@ from .exception_mapping_utils import exception_type, with_deployment_api_base from .llm_response_utils.get_api_base import get_api_base from .rules import Rules +if TYPE_CHECKING: + from .litellm_logging import BilledRoundsSpend + # Constants for special delta attribute names AUDIO_ATTRIBUTE: Final = "audio" IMAGE_ATTRIBUTE: Final = "images" @@ -259,6 +262,8 @@ def _provider_hidden_params( class CustomStreamWrapper: + _earlier_rounds_spend: "BilledRoundsSpend | None" = None + @property def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation return self._hidden_params @@ -2399,6 +2404,9 @@ class CustomStreamWrapper: return await asyncio.wait((self._failure_logging,), timeout=timeout) + def bill_failure_after_earlier_rounds(self, earlier: "BilledRoundsSpend") -> None: + self._earlier_rounds_spend = earlier + def _record_partial_usage_for_failure(self) -> None: """ A stream that breaks mid-flight still billed the provider for the chunks @@ -2425,10 +2433,12 @@ class CustomStreamWrapper: if self.model: partial_response.model = self.model backfill_missing_cache_usage_fields(usage) - self.logging_obj.model_call_details["combined_usage_object"] = usage - self.logging_obj.model_call_details["response_cost"] = ( - self.logging_obj.response_cost_calculator(result=partial_response) or 0.0 - ) + partial_cost: Final = self.logging_obj.response_cost_calculator(result=partial_response) or 0.0 + if self._earlier_rounds_spend is None: + self.logging_obj.model_call_details["combined_usage_object"] = usage + self.logging_obj.model_call_details["response_cost"] = partial_cost + return + self.logging_obj.record_failure_after_earlier_rounds(self._earlier_rounds_spend, usage, partial_cost) except Exception as recover_error: verbose_logger.debug( "could not recover partial usage for interrupted stream: %s", diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index cf616873a2e..6fda5780f63 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -67,6 +67,7 @@ from litellm.litellm_core_utils.get_supported_openai_params import ( get_supported_openai_params, ) from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params +from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost from litellm.litellm_core_utils.llm_response_utils.get_headers import ( @@ -3996,6 +3997,8 @@ class ProxyBaseLLMRequestProcessing: async def _on_deferred_native_stream_complete(*args: object) -> None: match args: + case (BilledRoundsDispatch(dispatch=logging_coroutine),): + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine) case (logging_coroutine,) if asyncio.iscoroutine(logging_coroutine): GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=logging_coroutine) case (assembled_response, cache_hit): diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index 4f2f7f9c256..a8ba8d65785 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -695,6 +695,7 @@ class RequestRateLimiterStash: owner_litellm_call_id: str | None = None rate_limit_response: RateLimitResponse | None = None parallel_slot: ParallelSlotAcquisition | None = None + parallel_slot_reused_at: datetime | None = None parallel_slot_release_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False, compare=False) reserved_tokens: int = 0 reserved_model: RateLimitedModel | None = None @@ -791,6 +792,19 @@ def claim_request_stash_for_data(data: dict) -> RequestRateLimiterStash: return stash +def _ended_before_slot_reuse(ended_at: datetime | None, reused_at: datetime | None) -> bool: + if ended_at is None or reused_at is None or (ended_at.tzinfo is None) != (reused_at.tzinfo is None): + return False + return ended_at <= reused_at + + +def _callback_kwargs_mapping(kwargs: object) -> Mapping[str, object]: + try: + return _REQUEST_RATE_LIMIT_DATA.validate_python(kwargs) + except ValidationError: + return MappingProxyType({}) + + def get_request_stash_for_call(litellm_call_id: str | None) -> RequestRateLimiterStash | None: stash: Final = _request_stash.get() if stash is None: @@ -1652,6 +1666,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): read_only: bool = False, skip_tpm_check: bool = False, parallel_slot_id: str | None = None, + skip_parallel_check: bool = False, ) -> RateLimitResponse: """ Check if any of the rate limit descriptors should be rate limited. @@ -1668,6 +1683,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ``reserve_tpm_tokens`` reservation path should set this to avoid the +1-per-key Lua / in-memory increment double-charging the tokens counter. + skip_parallel_check: If True, ``max_parallel_requests`` gauges are + neither checked nor acquired, for a call that runs inside a + request already holding its slot. ``max_parallel_requests`` descriptors are enforced by the dedicated concurrency-gauge path (``_check_parallel_request_gauges``), never by @@ -1760,7 +1778,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): if windowed_response["overall_code"] == "OVER_LIMIT": return windowed_response - if not gauges: + if not gauges or skip_parallel_check: return windowed_response gauge_response: Final = await self._check_parallel_request_gauges( @@ -2023,12 +2041,13 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): parent_otel_span: Span | None, *, in_logging_callback: bool = False, + round_ended_at: datetime | None = None, ) -> None: if stash is None: return async with stash.parallel_slot_release_lock: acquisition: Final = stash.parallel_slot - if acquisition is None: + if acquisition is None or _ended_before_slot_reuse(round_ended_at, stash.parallel_slot_reused_at): return deferred: Final = in_logging_callback and await self._defer_parallel_slot_release( acquisition, parent_otel_span @@ -3947,6 +3966,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): tpm_reservation_amount: int, call_type: str | None = None, endpoint_type: EndpointType = EndpointType.GENERIC, + reuses_request_slot: bool = False, ) -> None: """ Reserve project-scoped ITPM/OTPM tokens (Bedrock Mantle-style @@ -4026,7 +4046,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): parent_otel_span=user_api_key_dict.parent_otel_span, ) stash.reservation_released = True - await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span) + if not reuses_request_slot: + await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span) self._handle_rate_limit_error( response=io_response, descriptors=descriptors, @@ -4187,6 +4208,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): verbose_proxy_logger.debug("Inside Rate Limit Pre-Call Hook") stash: Final = claim_request_stash_for_data(data) + reuses_request_slot: Final = call_type == CallTypes.call_mcp_tool.value and stash.parallel_slot is not None + if reuses_request_slot: + stash.parallel_slot_reused_at = self._get_current_time() ######################################################### # Check if the call type has a specific rate limiter @@ -4231,11 +4255,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): # then because _reserve_project_io_tokens_or_raise below charges # them unconditionally and counting them here too would # double-charge every request. - parallel_counter_keys: Final = [ - self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests") - for d in descriptors - if (d.get("rate_limit") or {}).get("max_parallel_requests") is not None - ] + parallel_counter_keys: Final = ( + [] + if reuses_request_slot + else [ + self.create_rate_limit_keys(d["key"], d["value"], "max_parallel_requests") + for d in descriptors + if (d.get("rate_limit") or {}).get("max_parallel_requests") is not None + ] + ) parallel_slot_id: Final = uuid.uuid4().hex if parallel_counter_keys else None first_pass_descriptors: Final = ( @@ -4250,6 +4278,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): parent_otel_span=user_api_key_dict.parent_otel_span, skip_tpm_check=self.tpm_reservation_enabled, parallel_slot_id=parallel_slot_id, + skip_parallel_check=reuses_request_slot, ) if response["overall_code"] == "OVER_LIMIT": @@ -4259,7 +4288,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): requested_model=requested_model, ) else: - stash.rate_limit_response = response + if not reuses_request_slot: + stash.rate_limit_response = response if parallel_slot_id is not None: stash.parallel_slot = ParallelSlotAcquisition( slot_id=parallel_slot_id, @@ -4357,7 +4387,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): raw_estimate=estimated_tokens, ) if stash.ptu_ceiling is not None and ptu_estimated_tokens > stash.ptu_ceiling.tpm_limit: - await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span) + if not reuses_request_slot: + await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span) raise PTUShareExceededError(stash.ptu_ceiling, ptu_estimated_tokens) tpm_response: Final = await self.reserve_tpm_tokens( descriptors=descriptors, @@ -4367,7 +4398,8 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): ) if tpm_response["overall_code"] == "OVER_LIMIT": - await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span) + if not reuses_request_slot: + await self._release_stashed_parallel_slot(stash, user_api_key_dict.parent_otel_span) self._handle_rate_limit_error( response=tpm_response, descriptors=descriptors, @@ -4418,6 +4450,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): tpm_reservation_amount=tpm_reservation_amount, call_type=call_type, endpoint_type=endpoint_type, + reuses_request_slot=reuses_request_slot, ) def _create_pipeline_operations( @@ -5332,8 +5365,16 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): try: verbose_proxy_logger.debug("INSIDE parallel request limiter ASYNC SUCCESS LOGGING") - stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(kwargs)) - await self._release_stashed_parallel_slot(stash, litellm_parent_otel_span, in_logging_callback=True) + callback_kwargs: Final = _callback_kwargs_mapping(kwargs) + stash: Final = get_request_stash_for_call(_call_id_from_callback_kwargs(callback_kwargs)) + await self._release_stashed_parallel_slot( + stash, + litellm_parent_otel_span, + in_logging_callback=True, + round_ended_at=end_time + if isinstance(end_time, datetime) and not callback_kwargs.get("stream") + else None, + ) pipeline_operations: Final = self._build_success_event_pipeline_operations( kwargs=kwargs, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ea1044d4a55..6c9b23860da 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -158,7 +158,7 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int -from litellm.litellm_core_utils.litellm_logging import Logging, deployment_pricing_model_info +from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -4732,19 +4732,25 @@ class ProxyLogging: @staticmethod def _discard_deferred_stream_logging_for_failure(request_data: Mapping[str, object], error: Exception) -> bool: - """Drop the parked success dispatch for an assembled chat stream that ends in an error - ``post_call_failure_hook`` logs as a failure, billing its usage on the failure row instead. + """Drop the parked success dispatch for an assembled chat stream or a stream that billed several rounds when it + ends in an error ``post_call_failure_hook`` logs as a failure, billing its usage on the failure row instead. Returns False when the parked dispatch should still be flushed by the caller.""" logging_obj: Final = request_data.get("litellm_logging_obj") if not isinstance(logging_obj, Logging): return False _args: Final[tuple[object, ...] | None] = getattr(logging_obj, "_deferred_stream_complete_args", None) - assembled: Final = _args[0] if _args else None - if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS) or not isinstance(assembled, ModelResponse): + if not isinstance(error, _PROXY_ONLY_LLM_API_ERRORS): return False + match _args: + case (ModelResponse() as assembled, *_): + logging_obj.record_assembled_response_for_failure(assembled) + case (BilledRoundsDispatch() as billed,): + billed.dispatch.close() + logging_obj.record_billed_rounds_for_failure(billed.spend) + case _: + return False logging_obj._on_deferred_stream_complete = None logging_obj._deferred_stream_complete_args = None - logging_obj.record_assembled_response_for_failure(assembled) return True async def arelease_max_parallel_requests_on_disconnect( diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 84d4648f75f..4ef68aa072a 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -5,7 +5,8 @@ from itertools import filterfalse from typing import Any, Final, cast import litellm -from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params +from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache +from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.main import stream_chunk_builder from litellm.responses.litellm_completion_transformation.custom_tools import ( build_tool_call_item_kwargs, @@ -1363,6 +1364,17 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return False return not any(getattr(part, "text", None) for part in getattr(item, "content", None) or ()) + def finished_round_event(self) -> ResponseCompletedEvent | ResponseIncompleteEvent | None: + if not isinstance(self.litellm_model_response, ModelResponse): + return None + finished: Final = self._emit_terminal_response_event(self.litellm_model_response) + if finished is not None and served_from_cache(self.litellm_custom_stream_wrapper): + finished.response.hidden_params = {**finished.response.hidden_params, "cache_hit": True} + return finished + + def bill_failure_after_earlier_rounds(self, earlier: BilledRoundsSpend) -> None: + self.litellm_custom_stream_wrapper.bill_failure_after_earlier_rounds(earlier) + def partial_response_on_disconnect(self) -> ResponseIncompleteEvent | None: """ The response.incomplete event to bill when the client went away mid-stream. diff --git a/litellm/responses/mcp/mcp_streaming_iterator.py b/litellm/responses/mcp/mcp_streaming_iterator.py index 73cfba8dca4..030911238ec 100644 --- a/litellm/responses/mcp/mcp_streaming_iterator.py +++ b/litellm/responses/mcp/mcp_streaming_iterator.py @@ -3,7 +3,9 @@ from typing import TYPE_CHECKING, Any, Final, cast from litellm._logging import verbose_logger from litellm._uuid import uuid +from litellm.responses.litellm_completion_transformation.streaming_iterator import LiteLLMCompletionStreamingIterator from litellm.responses.mcp.request_context import MCPRequestContext +from litellm.responses.mcp.stream_round_billing import StreamedRoundBilling from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ( BaseLiteLLMOpenAIResponseObject, @@ -352,6 +354,9 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self._round_max_output_index = -1 self._composed_output: list[object] = [] # mutable-ok: grows as each round finishes self._pending_mcp_call_items: list[dict[str, object]] = [] # mutable-ok: grows per executed tool + self._iteration_started = False + self._round_billing: StreamedRoundBilling | None = None + self._bridged_round_billed: LiteLLMCompletionStreamingIterator | None = None def _extract_mcp_headers_from_params(self) -> None: """Extract MCP headers from original request params to pass to tool calls""" @@ -436,7 +441,15 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): return self async def __anext__(self) -> ResponsesAPIStreamingResponse: - chunk: Final = await self._anext_impl() + if not self._iteration_started: + self._iteration_started = True + self._round_billing = self._bill_rounds_on_one_row() + try: + chunk: Final = await self._anext_impl() + except Exception: + if self._round_billing is not None: + self._round_billing.finish() + raise sequence_number: Final = getattr(chunk, "sequence_number", None) if isinstance(sequence_number, int): if sequence_number <= self._last_sequence_number and self._last_sequence_number > 0: @@ -446,6 +459,49 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self._last_sequence_number = max(self._last_sequence_number, sequence_number) return chunk + def __del__(self) -> None: + round_billing: Final[object] = getattr(self, "_round_billing", None) + if isinstance(round_billing, StreamedRoundBilling): + round_billing.release_abandoned() + + def _bill_rounds_on_one_row(self) -> StreamedRoundBilling | None: + if not self.should_auto_execute: + return None + if isinstance(self.base_iterator, LiteLLMCompletionStreamingIterator): + return StreamedRoundBilling(self.base_iterator.litellm_logging_obj) + if isinstance(self.base_iterator, BaseResponsesAPIStreamingIterator): + return StreamedRoundBilling(self.base_iterator.logging_obj) + return None + + def _restore_round_headers(self, logged_event: ResponsesAPIStreamingResponse) -> None: + if isinstance(self.base_iterator, LiteLLMCompletionStreamingIterator): + return + if isinstance(self.base_iterator, BaseResponsesAPIStreamingIterator): + self.base_iterator._restore_provider_response_headers(logged_event) + + def _round_billed_early(self) -> bool: + return self._bridged_round_billed is not None and self._bridged_round_billed is self.base_iterator + + def _bill_finished_bridged_round(self) -> None: + base: Final = self.base_iterator + if self._round_billing is None or not isinstance(base, LiteLLMCompletionStreamingIterator): + return + if self._round_billed_early(): + return + finished: Final = base.finished_round_event() + if finished is None: + return + self._bridged_round_billed = base + round_output: Final = _output_items(finished.response) + self._round_billing.complete_round( + finished, + finished.response, + round_output + if finished.type == ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE + else (*self._composed_output, *round_output), + self._restore_round_headers, + ) + async def _anext_impl(self) -> ResponsesAPIStreamingResponse: """ Phase-based streaming: @@ -617,6 +673,7 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): follow-up rounds past the items already emitted, and lists every round's items on the final response.completed. """ + self._bill_finished_bridged_round() chunk_type: Final[object] = getattr(chunk, "type", None) if self._round_index > 0 and chunk_type in ( ResponsesAPIStreamEvents.RESPONSE_CREATED, @@ -630,12 +687,21 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): if self._output_index_offset: _set_event_field(chunk, "output_index", output_index + self._output_index_offset) + if self.should_auto_execute and chunk_type == ResponsesAPIStreamEvents.RESPONSE_INCOMPLETE: + return self._charge_incomplete_round(chunk) if not (self.should_auto_execute and self._is_response_completed(chunk)): return chunk response_obj: Final[object] = getattr(chunk, "response", None) if isinstance(response_obj, ResponsesAPIResponse): self.collected_response = response_obj + if self._round_billing is not None and not self._round_billed_early(): + self._round_billing.complete_round( + chunk, + response_obj, + (*self._composed_output, *_output_items(response_obj)), + self._restore_round_headers, + ) # Move to tool execution phase after this chunk self.phase = "tool_execution" await self._generate_tool_execution_events() @@ -645,13 +711,33 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): if self._follow_up_pending(): self._absorb_round(response_obj) return None - if self._composed_output: - merged_output: Final[list[object]] = [ # mutable-ok: the response model declares output as a list - *self._composed_output, - *_output_items(response_obj), - ] - merged_response: Final = response_obj.model_copy(update={"output": merged_output}) - _set_event_field(chunk, "response", merged_response) + merged_response: Final = ( + response_obj.model_copy(update={"output": [*self._composed_output, *_output_items(response_obj)]}) + if self._composed_output + else response_obj + ) + client_response: Final = ( + merged_response + if self._round_billing is None + else self._round_billing.charge_final_round(chunk, merged_response, self._restore_round_headers) + ) + if client_response is not response_obj: + _set_event_field(chunk, "response", client_response) + return chunk + + def _charge_incomplete_round(self, chunk: ResponsesAPIStreamingResponse) -> ResponsesAPIStreamingResponse: + response_obj: Final[object] = getattr(chunk, "response", None) + if self._round_billing is None or not isinstance(response_obj, ResponsesAPIResponse): + return chunk + if not self._round_billed_early(): + self._round_billing.complete_round( + chunk, response_obj, _output_items(response_obj), self._restore_round_headers + ) + client_response: Final = self._round_billing.charge_final_round( + chunk, response_obj, self._restore_round_headers + ) + if client_response is not response_obj: + _set_event_field(chunk, "response", client_response) return chunk async def _process_base_iterator_chunk(self) -> ResponsesAPIStreamingResponse: @@ -949,6 +1035,8 @@ class MCPEnhancedStreamingIterator(BaseResponsesAPIStreamingIterator): self.collected_response = None self._cached_response_id = None self._round_index += 1 + if self._round_billing is not None: + self._round_billing.record_billed_rounds(follow_up_response) except Exception as e: verbose_logger.error("Error creating follow-up iterator: %s", e) diff --git a/litellm/responses/mcp/round_billing.py b/litellm/responses/mcp/round_billing.py index ae34ab31633..93769fbe82a 100644 --- a/litellm/responses/mcp/round_billing.py +++ b/litellm/responses/mcp/round_billing.py @@ -4,7 +4,7 @@ from contextlib import contextmanager from types import MappingProxyType from typing import Final, TypeVar -from pydantic import TypeAdapter +from pydantic import BaseModel, TypeAdapter from pydantic_core import to_jsonable_python from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_params @@ -46,15 +46,18 @@ def _summed_mapping(first: Mapping[str, object], final: Mapping[str, object]) -> return MappingProxyType({key: _summed_entry(key, first, final) for key in keys}) +def _summed_usage(first: BaseModel, final: BaseModel) -> object: + return to_jsonable_python(_summed_mapping(first.model_dump(), final.model_dump()), fallback=dict) + + def merged_round_usage(first: ResponseAPIUsage | None, final: ResponseAPIUsage | None) -> ResponseAPIUsage | None: if first is None or final is None: return final if first is None else first - return ResponseAPIUsage.model_validate(_summed_mapping(first.model_dump(), final.model_dump())) + return ResponseAPIUsage.model_validate(_summed_usage(first, final)) -def _merged_chat_usage(first: Usage, final: Usage) -> Usage: - summed: Final = _summed_mapping(first.model_dump(), final.model_dump()) - return Usage.model_validate(to_jsonable_python(summed, fallback=dict)) +def merged_chat_usage(first: Usage, final: Usage) -> Usage: + return Usage.model_validate(_summed_usage(first, final)) def _chat_usage(round_response: ResponsesAPIResponse | ModelResponse) -> Usage | None: @@ -82,7 +85,7 @@ def _usage_of_both_rounds( final_usage: Final = _chat_usage(final) if first_usage is None or final_usage is None: return final_usage if first_usage is None else first_usage - return _merged_chat_usage(first_usage, final_usage) + return merged_chat_usage(first_usage, final_usage) def _repeated_fixed_margin( @@ -105,6 +108,14 @@ def _less_repeated_fixed_margin(summed: Mapping[str, object], repeated: float) - return MappingProxyType({**summed, **charged_once}) +def breakdown_less_repeated_fixed_margin( + breakdown: CostBreakdown | None, repeated_fixed_margin: float +) -> CostBreakdown | None: + if breakdown is None: + return None + return _COST_BREAKDOWN.validate_python(_less_repeated_fixed_margin(breakdown, repeated_fixed_margin)) + + def summed_cost_breakdown( first: ResponsesAPIResponse | ModelResponse, first_breakdown: CostBreakdown | None, diff --git a/litellm/responses/mcp/stream_round_billing.py b/litellm/responses/mcp/stream_round_billing.py new file mode 100644 index 00000000000..443147230bf --- /dev/null +++ b/litellm/responses/mcp/stream_round_billing.py @@ -0,0 +1,376 @@ +import asyncio +from collections.abc import Callable, Coroutine, Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Final, Protocol, TypeAlias + +from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache +from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, BilledRoundsSpend, UncachedPrice +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.responses.mcp.round_billing import billed_for_every_round, merged_chat_usage, summed_cost_breakdown +from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator, spawn_logging_task +from litellm.responses.utils import ResponseAPILoggingUtils +from litellm.types.llms.openai import ResponsesAPIResponse, ResponsesAPIStreamingResponse +from litellm.types.utils import CostBreakdown, ModelResponse, Usage + +Parked: TypeAlias = tuple[object, ...] +RestoresHeaders: TypeAlias = Callable[[ResponsesAPIStreamingResponse], None] + +_PARKED_DISPATCH: Final = "_deferred_stream_complete_args" +_DISPATCH_RELEASE: Final = "_on_deferred_stream_complete" +_SERVED_FROM_CACHE: Final = UncachedPrice(cost=0.0, cost_breakdown=None) + + +class _DispatchesSuccessHandlers(Protocol): + async def dispatch_success_handlers( + self, + result: object = None, + start_time: object = None, + end_time: object = None, + cache_hit: object = None, + prefer_async_handlers: bool = False, + ) -> None: ... + + +def _success_dispatcher(logger: LiteLLMLoggingObj) -> _DispatchesSuccessHandlers: + return logger + + +def _event_loop_running() -> bool: + try: + asyncio.get_running_loop() + except RuntimeError: + return False + return True + + +def _priced(response: ResponsesAPIResponse, price: UncachedPrice) -> ResponsesAPIResponse: + priced: Final = response.model_copy() + priced.hidden_params = {**response.hidden_params, "response_cost": price.cost} + return priced + + +def _assembled(parked: Parked | None) -> ModelResponse | None: + match parked: + case (ModelResponse() as assembled, bool()): + return assembled + case _: + return None + + +def _bridged_round_usage(parked: Parked | None) -> Usage | None: + usage: Final[object] = getattr(_assembled(parked), "usage", None) + return usage if isinstance(usage, Usage) else None + + +def _usage_of_bridged_rounds(billed: Usage | None, round_usage: Usage | None, cost: object) -> Usage | None: + if billed is None or round_usage is None: + return None + return merged_chat_usage(billed, round_usage).model_copy(update={"cost": cost}) + + +def _logged_cache_hit(response: ResponsesAPIResponse, chat_usage: Usage | None) -> bool | None: + if chat_usage is not None: + return served_from_cache(response) + return served_from_cache(response) or None + + +@dataclass(frozen=True, slots=True) +class _RoundRow: + logged: ResponsesAPIStreamingResponse | ModelResponse + cache_hit: bool | None + end_time: datetime + + +@dataclass(frozen=True, slots=True) +class _SuccessLog: + logger: LiteLLMLoggingObj + row: _RoundRow + chat_usage: Usage | None + + def carry_bridged_usage(self) -> None: + if self.chat_usage is not None: + self.logger.model_call_details["combined_usage_object"] = self.chat_usage + + +def _round_row( + completed: ResponsesAPIStreamingResponse, + response: ResponsesAPIResponse, + chat_usage: Usage | None, + restore_headers: RestoresHeaders, +) -> _RoundRow: + logged_response: Final = response.model_copy() + logged_response.hidden_params = {**response.hidden_params} + logged: Final = completed.model_copy(update={"response": logged_response}) + restore_headers(logged) + return _RoundRow( + logged, + _logged_cache_hit(response, chat_usage), + datetime.now(), # noqa: DTZ005 # naive to match the logging pipeline start_time + ) + + +@dataclass(frozen=True, slots=True) +class _BilledRounds: + response: ResponsesAPIResponse + breakdown: CostBreakdown | None + chat_usage: Usage | None + rounds: int = 1 + + @property + def stamped_cost(self) -> float | None: + cost: Final = self.response.hidden_params.get("response_cost") + return float(cost) if isinstance(cost, (int, float)) else None + + @property + def cost(self) -> float: + return self.stamped_cost or 0.0 + + @property + def spend(self) -> BilledRoundsSpend: + usage: Final = ( + ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(self.response.usage) + if self.chat_usage is None + else self.chat_usage + ) + return BilledRoundsSpend(usage, self.cost, self.breakdown) + + def with_round( + self, priced: ResponsesAPIResponse, breakdown: CostBreakdown | None, chat_usage: Usage | None + ) -> "_BilledRounds": + merged: Final = billed_for_every_round(self.response, self.breakdown, priced, breakdown) + total: Final = merged.hidden_params.get("response_cost") + charged: Final = merged.model_copy( + update={"usage": None if merged.usage is None else merged.usage.model_copy(update={"cost": total})} + ) + charged.hidden_params = { + **merged.hidden_params, + "cache_hit": served_from_cache(self.response) and served_from_cache(priced), + } + return _BilledRounds( + charged, + summed_cost_breakdown(self.response, self.breakdown, priced, breakdown), + _usage_of_bridged_rounds(self.chat_usage, chat_usage, total), + self.rounds + 1, + ) + + def row( + self, + completed: ResponsesAPIStreamingResponse, + client_output: Sequence[object], + restore_headers: RestoresHeaders, + ) -> _RoundRow: + return _round_row( + completed, + self.response.model_copy(update={"output": list(client_output)}), + self.chat_usage, + restore_headers, + ) + + def bridged_row(self, assembled: ModelResponse | None, logged_id: str) -> _RoundRow | None: + if assembled is None or self.chat_usage is None: + return None + logged: Final = assembled.model_copy(update={"id": logged_id, "usage": self.chat_usage}) + logged.hidden_params = { + **(get_hidden_params(assembled) or {}), + "response_cost": self.stamped_cost, + } + return _RoundRow( + logged, + served_from_cache(self.response), + datetime.now(), # noqa: DTZ005 # naive to match the logging pipeline start_time + ) + + +class StreamedRoundBilling: + """One success row per streamed auto-execute request, billed for every round it ran. + + Each round's iterator parks its success dispatch on the shared logging object rather than sending it, either as a + ready dispatch or, for a stream bridged through chat completions, as the assembled response and its cache hit. + When a round completes, before its tools run, it is folded into the rounds before it and a dispatch for every + round so far stays parked, so a client disconnect or a later round that fails still bills them. Once a later round + starts, that dispatch logs from a copy of the logging state taken then, so the later round's failure, logged on the + request's own state where the proxy reads its failure row, never reaches their success row. Only the latest + dispatch is sent, through the proxy's deferred release when the proxy armed one.""" + + def __init__(self, logging_obj: LiteLLMLoggingObj) -> None: + self._logging_obj: Final = logging_obj + self._billed: _BilledRounds | None = None + self._banked: Parked | None = None + self._banked_row: _RoundRow | None = None + if getattr(logging_obj, _DISPATCH_RELEASE, None) is None: + setattr(logging_obj, _DISPATCH_RELEASE, self._run_parked) + + def complete_round( + self, + completed: ResponsesAPIStreamingResponse, + response: ResponsesAPIResponse, + client_output: Sequence[object], + restore_headers: RestoresHeaders, + ) -> None: + round_parked: Final = self._take_parked() + price: Final = self._price(response, round_parked) + priced: Final = _priced(response, price) + chat_usage: Final = _bridged_round_usage(round_parked) + if self._billed is None: + first_round: Final = _BilledRounds(priced, price.cost_breakdown, chat_usage) + self._billed = first_round + self._bank( + round_parked, + first_round.bridged_row(_assembled(round_parked), priced.id) + or first_round.row(completed, client_output, restore_headers), + ) + return + billed: Final = self._billed.with_round(priced, price.cost_breakdown, chat_usage) + self._close(round_parked) + self._close(self._banked) + self._logging_obj.model_call_details.pop("combined_usage_object", None) + self._logging_obj.model_call_details.pop("response_cost", None) + self._billed = billed + self._logging_obj.cost_breakdown = billed.breakdown + row: Final = billed.bridged_row(_assembled(round_parked), billed.response.id) or billed.row( + completed, client_output, restore_headers + ) + self._bank(None if round_parked is None else (self._dispatch(row, billed, self._logging_obj),), row) + + def record_billed_rounds(self, follow_up: object) -> None: + if self._billed is None or not isinstance(follow_up, BaseResponsesAPIStreamingIterator): + return + follow_up.bill_failure_after_earlier_rounds(self._billed.spend) + self._bank_apart_from_later_rounds(self._billed) + + def charge_final_round( + self, completed: ResponsesAPIStreamingResponse, response: ResponsesAPIResponse, restore_headers: RestoresHeaders + ) -> ResponsesAPIResponse: + parked: Final = self._take_parked() + self._banked = None + if self._billed is None or self._billed.rounds == 1: + if parked is not None: + self._release(parked) + return response + self._close(parked) + charged: Final = response.model_copy(update={"usage": self._billed.response.usage}) + charged.hidden_params = {**self._billed.response.hidden_params} + if parked is not None: + final_row: Final = self._billed.bridged_row(self._latest_assembled(), charged.id) or _round_row( + completed, charged, self._billed.chat_usage, restore_headers + ) + self._release((self._dispatch(final_row, self._billed, self._logging_obj),)) + return charged + + def finish(self) -> None: + parked: Final = self._take_parked() + self._banked = None + if parked is not None: + self._release(parked) + return + self._disarm() + + def release_abandoned(self) -> None: + if not self._owns_release(): + return + if _event_loop_running(): + self.finish() + return + self.abandon() + + def abandon(self) -> None: + self._close(self._take_parked()) + self._close(self._banked) + self._banked = None + self._disarm() + + def _latest_assembled(self) -> ModelResponse | None: + row: Final = self._banked_row + return row.logged if row is not None and isinstance(row.logged, ModelResponse) else None + + def _price(self, response: ResponsesAPIResponse, round_parked: Parked | None) -> UncachedPrice: + if served_from_cache(response): + return _SERVED_FROM_CACHE + match round_parked: + case (ModelResponse() as assembled, bool()): + return self._logging_obj.uncached_price(assembled) + case _: + return self._logging_obj.uncached_price(response) + + def _bank(self, parked: Parked | None, row: _RoundRow) -> None: + self._banked = parked + self._banked_row = row + if parked is not None: + setattr(self._logging_obj, _PARKED_DISPATCH, parked) + + def _bank_apart_from_later_rounds(self, billed: _BilledRounds) -> None: + row: Final = self._banked_row + if row is None: + return + parked: Final = self._take_parked() + if parked is None: + return + self._close(parked) + self._bank((self._dispatch(row, billed, self._logging_obj.attempt_scoped_copy()),), row) + + def _dispatch( + self, row: _RoundRow, billed: _BilledRounds, logger: LiteLLMLoggingObj + ) -> BilledRoundsDispatch | Coroutine[object, object, None]: + dispatch: Final = self._log_success(_SuccessLog(logger, row, billed.chat_usage)) + if billed.chat_usage is None: + return dispatch + return BilledRoundsDispatch(dispatch, billed.spend) + + @staticmethod + async def _log_success(success: _SuccessLog) -> None: + success.carry_bridged_usage() + await _success_dispatcher(success.logger).dispatch_success_handlers( + success.row.logged, + start_time=success.logger.start_time, + end_time=success.row.end_time, + cache_hit=success.row.cache_hit, + prefer_async_handlers=True, + ) + + def _take_parked(self) -> Parked | None: + parked: Final[Parked | None] = getattr(self._logging_obj, _PARKED_DISPATCH, None) + if not isinstance(parked, tuple): + return None + setattr(self._logging_obj, _PARKED_DISPATCH, None) + return parked + + async def _run_parked(self, *parked: object) -> None: + match parked: + case (BilledRoundsDispatch(dispatch=dispatch),): + await dispatch + case (dispatch,) if asyncio.iscoroutine(dispatch): + await dispatch + case (assembled, cache_hit): + await _success_dispatcher(self._logging_obj).dispatch_success_handlers( + assembled, cache_hit=cache_hit, start_time=None, end_time=None, prefer_async_handlers=True + ) + case _: + return + + def _owns_release(self) -> bool: + release: Final[object] = getattr(self._logging_obj, _DISPATCH_RELEASE, None) + return release == self._run_parked + + def _release(self, parked: Parked) -> None: + if getattr(self._logging_obj, _DISPATCH_RELEASE, None) is not None and not self._owns_release(): + setattr(self._logging_obj, _PARKED_DISPATCH, parked) + return + setattr(self._logging_obj, _DISPATCH_RELEASE, None) + spawn_logging_task( + asyncio.get_running_loop(), self._run_parked(*parked), task_name="Responses MCP stream success logging" + ) + + def _disarm(self) -> None: + if self._owns_release(): + setattr(self._logging_obj, _DISPATCH_RELEASE, None) + + @staticmethod + def _close(parked: Parked | None) -> None: + match parked: + case (BilledRoundsDispatch(dispatch=dispatch),): + dispatch.close() + case (dispatch,) if asyncio.iscoroutine(dispatch): + dispatch.close() + case _: + return diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index c5dc45566a9..bc30da72df9 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -42,6 +42,7 @@ from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.exception_mapping_utils import with_deployment_api_base from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation +from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers @@ -363,6 +364,7 @@ class BaseResponsesAPIStreamingIterator: """ _pending_logging_tasks: tuple[asyncio.Task[object], ...] = () + _earlier_rounds_spend: BilledRoundsSpend | None = None def __init__( self, @@ -705,18 +707,22 @@ class BaseResponsesAPIStreamingIterator: if usage_obj is None: return try: - self.logging_obj.model_call_details["combined_usage_object"] = ( - ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj) - ) + failed_usage: Final = ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage_obj) except (TypeError, ValueError) as usage_error: verbose_logger.debug( "could not record usage for failed responses stream: %s", usage_error, ) return - self.logging_obj.model_call_details["response_cost"] = ( - self.logging_obj.response_cost_calculator(result=response_obj) or 0.0 - ) + failed_cost: Final = self.logging_obj.response_cost_calculator(result=response_obj) or 0.0 + if self._earlier_rounds_spend is None: + self.logging_obj.model_call_details["combined_usage_object"] = failed_usage + self.logging_obj.model_call_details["response_cost"] = failed_cost + return + self.logging_obj.record_failure_after_earlier_rounds(self._earlier_rounds_spend, failed_usage, failed_cost) + + def bill_failure_after_earlier_rounds(self, earlier: BilledRoundsSpend) -> None: + self._earlier_rounds_spend = earlier def _map_error_event_exception(self, error_obj: object) -> Exception: return _map_stream_error_to_exception( diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 4f8a13c1182..43190852c60 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -42,6 +42,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging from litellm.litellm_core_utils.litellm_logging import ( + BilledRoundsSpend, Logging, StandardLoggingPayloadSetup, _extract_response_obj_and_hidden_params, @@ -6837,6 +6838,26 @@ def test_failure_handler_zeroes_spend_without_recovered_usage(logging_obj): assert payload["total_tokens"] == 0 +def test_failure_after_earlier_rounds_bills_only_the_failed_round_when_it_priced_no_breakdown( + logging_obj: LitellmLogging, +) -> None: + earlier_breakdown: Final = CostBreakdown(total_cost=0.3, margin_fixed_amount=0.1) + logging_obj.cost_breakdown = earlier_breakdown + logging_obj.model_call_details["litellm_params"] = {} + + logging_obj.record_failure_after_earlier_rounds( + BilledRoundsSpend(Usage(prompt_tokens=5, completion_tokens=2, total_tokens=7), 0.3, earlier_breakdown), + Usage(prompt_tokens=7, completion_tokens=0, total_tokens=7), + 0.0, + ) + logging_obj.failure_handler(litellm.ServiceUnavailableError("503", "anthropic", "claude"), "tb") + + row: Final = logging_obj.model_call_details["standard_logging_object"] + assert (row["prompt_tokens"], row["completion_tokens"]) == (7, 0) + assert row["response_cost"] == pytest.approx(0.0) + assert row.get("cost_breakdown") is None or row["cost_breakdown"]["total_cost"] == pytest.approx(0.0) + + def test_set_cost_breakdown_stores_reasoning_cost(): """reasoning_cost is stored only when positive, mirroring the cache-cost fields.""" from datetime import datetime diff --git a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py index 8a74f2d50b4..1ec505362dd 100644 --- a/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py +++ b/tests/unit/proxy/hooks/test_parallel_request_limiter_v3.py @@ -3,6 +3,7 @@ Unit Tests for the max parallel request limiter v3 for the proxy """ import asyncio +import contextvars import logging import sys import time @@ -54,6 +55,7 @@ from litellm.types.mcp import MCPPreCallRequestObject, MCPTransport from litellm.types.mcp_server.mcp_server_manager import MCPServer from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem from litellm.types.utils import ( + CallTypes, EmbeddingResponse, ModelResponse, PromptTokensDetailsWrapper, @@ -4206,6 +4208,175 @@ async def test_completed_responses_post_call_releases_parallel_slot() -> None: ) == 0 +@pytest.mark.asyncio +async def test_mcp_tool_call_inside_admitted_request_reuses_its_parallel_slot() -> None: + api_key: Final = hash_token("sk-mcp-auto-execute") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict: Final = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=1) + parallel_key: Final = f"{{api_key:{api_key}}}:max_parallel_requests" + owner_call_id: Final = "mcp-auto-execute-owner" + + async def in_flight() -> int: + return handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) + + def mcp_tool_call() -> dict[str, object]: + return { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "Tool: add"}], + "mcp_server_name": "math", + } + + async def pre_call(data: dict[str, object], call_type: str) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type=call_type, + ) + + async def pre_call_from_another_request( + data: dict[str, object], call_type: str + ) -> None: + await contextvars.Context().run(asyncio.create_task, pre_call(data, call_type)) + + await pre_call( + {"model": "gpt-4o-mini", "input": "add 2 and 3", "litellm_call_id": owner_call_id}, + CallTypes.aresponses.value, + ) + await pre_call(mcp_tool_call(), CallTypes.call_mcp_tool.value) + await pre_call(mcp_tool_call(), CallTypes.call_mcp_tool.value) + assert await in_flight() == 1 + + with pytest.raises(HTTPException): + await pre_call_from_another_request( + mcp_tool_call(), CallTypes.call_mcp_tool.value + ) + + await handler.async_log_success_event( + kwargs={"litellm_call_id": owner_call_id}, + response_obj=None, + start_time=None, + end_time=None, + ) + assert await in_flight() == 0 + await pre_call_from_another_request( + {"model": "gpt-4o-mini", "input": "next request"}, + CallTypes.aresponses.value, + ) + assert await in_flight() == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("stream", "in_flight_after_first_round_logs"), + [(False, 1), (True, 0)], + ids=["non_stream_round_logged_late_keeps_slot", "stream_logged_at_its_end_releases_slot"], +) +async def test_first_round_logged_after_tool_call_reuse_releases_only_at_request_end( + stream: bool, in_flight_after_first_round_logs: int +) -> None: + reused_at: Final = datetime(2026, 1, 5, 12, 0, 0) + api_key: Final = hash_token(f"sk-mcp-late-round-log-{stream}") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache), + time_provider=lambda: reused_at, + ) + user_api_key_dict: Final = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=1) + parallel_key: Final = f"{{api_key:{api_key}}}:max_parallel_requests" + owner_call_id: Final = f"mcp-late-round-log-{stream}" + + async def in_flight() -> int: + return handler._gauge_in_flight_from_cache_value( + await local_cache.async_get_cache(key=parallel_key) + ) + + async def pre_call(data: dict[str, object], call_type: str) -> None: + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=data, + call_type=call_type, + ) + + async def round_logged(ended_at: datetime) -> None: + await handler.async_log_success_event( + kwargs={"litellm_call_id": owner_call_id, "stream": stream}, + response_obj=None, + start_time=None, + end_time=ended_at, + ) + + await pre_call( + { + "model": "claude-haiku-5-5", + "messages": [{"role": "user", "content": "add 2 and 3"}], + "litellm_call_id": owner_call_id, + }, + CallTypes.anthropic_messages.value, + ) + await pre_call( + {"model": "MCP: add", "messages": [{"role": "user", "content": "Tool: add"}], "mcp_server_name": "math"}, + CallTypes.call_mcp_tool.value, + ) + + await round_logged(reused_at - timedelta(seconds=1)) + assert await in_flight() == in_flight_after_first_round_logs + + await round_logged(reused_at + timedelta(seconds=1)) + assert await in_flight() == 0 + + +@pytest.mark.asyncio +async def test_mcp_tool_call_reusing_request_slot_keeps_request_max_parallel_headers() -> None: + api_key: Final = hash_token("sk-mcp-reuse-headers") + local_cache: Final = DualCache() + handler: Final = _PROXY_MaxParallelRequestsHandler( + internal_usage_cache=InternalUsageCache(local_cache) + ) + user_api_key_dict: Final = UserAPIKeyAuth(api_key=api_key, max_parallel_requests=2) + data: Final[dict[str, object]] = { + "model": "gpt-4o-mini", + "input": "add 2 and 3", + "litellm_call_id": "mcp-reuse-headers-owner", + } + for request_data, call_type in ( + (data, CallTypes.aresponses.value), + ( + {"model": "MCP: add", "messages": [{"role": "user", "content": "Tool: add"}], "mcp_server_name": "math"}, + CallTypes.call_mcp_tool.value, + ), + ): + await handler.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=local_cache, + data=request_data, + call_type=call_type, + ) + response: Final = ModelResponse( + id="mcp-reuse-headers", + object="chat.completion", + created=0, + model="gpt-4o-mini", + usage=Usage(prompt_tokens=1, completion_tokens=1, total_tokens=2), + choices=[], + ) + response._hidden_params = {} + + await handler.async_post_call_success_hook( + data=data, user_api_key_dict=user_api_key_dict, response=response + ) + + headers: Final = response._hidden_params["additional_headers"] + assert headers["x-ratelimit-api_key-limit-max_parallel_requests"] == 2 + assert headers["x-ratelimit-api_key-remaining-max_parallel_requests"] == 1 + + @pytest.mark.asyncio async def test_concurrent_success_callbacks_release_parallel_slot_once_when_redis_fails() -> None: from unittest.mock import AsyncMock diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index 60d4d2e89b5..f0f3fc37ddb 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -1,7 +1,9 @@ import asyncio +import gc import json import logging import sys +import warnings from collections.abc import Callable, Iterator, Mapping, Sequence from contextlib import contextmanager, nullcontext from datetime import datetime @@ -18,9 +20,11 @@ from mcp.types import Tool as MCPTool # Import required modules import litellm +from litellm.exceptions import GuardrailRaisedException, MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.langfuse.langfuse import langfuse_trace_id, langfuse_trace_metadata from litellm.litellm_core_utils.litellm_logging import Logging +from litellm.llms.xai.common_utils import xai_reported_cost_in_usd from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk from litellm.proxy.utils import _failure_fields_to_lift @@ -31,7 +35,7 @@ from litellm.types.llms.openai import ( OpenAIMcpServerTool, ToolParam, ) -from litellm.types.utils import StandardLoggingPayload, Usage +from litellm.types.utils import ModelResponse, StandardLoggingPayload, Usage class MockUserAPIKeyAuth: @@ -1518,25 +1522,39 @@ def _openai_answer_round() -> bytes: ) -def _anthropic_round(message_id: str, block: Mapping[str, object], delta: Mapping[str, object], stop: str) -> bytes: +def _anthropic_message_start( + message_id: str, model: str, input_tokens: int, cache_usage: Mapping[str, object] = MappingProxyType({}) +) -> dict[str, object]: + return { + "type": "message_start", + "message": { + "id": message_id, + "type": "message", + "role": "assistant", + "model": model, + "content": [], + "stop_reason": None, + "usage": {"input_tokens": input_tokens, "output_tokens": 1, **cache_usage}, + }, + } + + +def _anthropic_round( + message_id: str, + block: Mapping[str, object], + delta: Mapping[str, object], + stop: str, + model: str = "claude-haiku-4-5", + usage: tuple[int, int] = (5, 5), + cache_usage: Mapping[str, object] = MappingProxyType({}), +) -> bytes: return _sse( ( - { - "type": "message_start", - "message": { - "id": message_id, - "type": "message", - "role": "assistant", - "model": "claude-haiku-4-5", - "content": [], - "stop_reason": None, - "usage": {"input_tokens": 5, "output_tokens": 1}, - }, - }, + _anthropic_message_start(message_id, model, usage[0], cache_usage), {"type": "content_block_start", "index": 0, "content_block": dict(block)}, {"type": "content_block_delta", "index": 0, "delta": dict(delta)}, {"type": "content_block_stop", "index": 0}, - {"type": "message_delta", "delta": {"stop_reason": stop}, "usage": {"output_tokens": 5}}, + {"type": "message_delta", "delta": {"stop_reason": stop}, "usage": {"output_tokens": usage[1]}}, {"type": "message_stop"}, ) ) @@ -1586,6 +1604,7 @@ class _RoundRecorder(CustomLogger): def __init__(self) -> None: super().__init__() self.logged: tuple[StandardLoggingPayload, ...] = () + self.results: tuple[object, ...] = () self.failed: tuple[StandardLoggingPayload, ...] = () self.hooked: tuple[object, ...] = () @@ -1593,6 +1612,7 @@ class _RoundRecorder(CustomLogger): self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime ) -> None: self.logged = (*self.logged, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + self.results = (*self.results, response_obj) async def async_log_failure_event( self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime @@ -1624,12 +1644,15 @@ _ECHO_MCP_TOOL: Final = MappingProxyType( def _serve_responses( - respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, responses: Sequence[httpx.Response] + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, + responses: Sequence[httpx.Response], + url_regex: str = r"https://mcp-rounds\.example\.com/.*responses", ) -> respx.Route: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") litellm.in_memory_llm_clients_cache.flush_cache() - return respx_mock.post(url__regex=r"https://mcp-rounds\.example\.com/.*responses").mock(side_effect=list(responses)) + return respx_mock.post(url__regex=url_regex).mock(side_effect=list(responses)) def _serve_rounds( @@ -2424,3 +2447,894 @@ async def test_streaming_mcp_event_order_and_response_id_consistency( assert lifecycle_ids[0] == lifecycle_ids[1] assert lifecycle_ids[2] != lifecycle_ids[0] assert not [record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR] + + +_FIXED_MARGIN: Final = 0.001 + + +def _streamed(body: Mapping[str, object], **terminal: object) -> httpx.Response: + completed: Final = json.loads(json.dumps({**body, **terminal}, default=dict)) + opened: Final = {**completed, "status": "in_progress", "output": [], "usage": None} + items: Final = tuple(enumerate(completed["output"])) + events: Final = ( + {"type": "response.created", "response": opened}, + {"type": "response.in_progress", "response": opened}, + *({"type": "response.output_item.added", "output_index": index, "item": item} for index, item in items), + *({"type": "response.output_item.done", "output_index": index, "item": item} for index, item in items), + {"type": f"response.{completed['status']}", "response": completed}, + ) + return httpx.Response( + 200, + content=_sse(tuple({**event, "sequence_number": number} for number, event in enumerate(events))), + headers=SSE_HEADERS, + ) + + +class _CutAfterTheFirstEvent(httpx.AsyncByteStream): + async def __aiter__(self): + opened: Final = {**json.loads(json.dumps(_ROUND_TWO_BODY, default=dict)), "status": "in_progress"} + yield _sse(({"type": "response.created", "response": opened, "sequence_number": 0},)) + raise httpx.ReadError("connection reset mid-stream") + + +async def _open_stream_auto_execute(logging_obj: Logging) -> object: + return await litellm.aresponses( + model="openai/gpt-5.6", + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + api_key="sk-mcp-rounds", + api_base="https://mcp-rounds.example.com/v1", + stream=True, + litellm_logging_obj=logging_obj, + ) + + +async def _drain(stream: object) -> tuple[ResponsesAPIStreamingResponse, ...]: + return tuple([event async for event in cast(Any, stream)]) + + +async def _stream_auto_execute(logging_obj: Logging) -> tuple[ResponsesAPIStreamingResponse, ...]: + with _echo_tool_served("PING"): + return await _drain(await _open_stream_auto_execute(logging_obj)) + + +def _final_response( + events: Sequence[ResponsesAPIStreamingResponse], terminal: str = "response.completed" +) -> ResponsesAPIResponse: + assert _event_type(events[-1]) == terminal + response: Final = getattr(events[-1], "response", None) + assert isinstance(response, ResponsesAPIResponse) + return response + + +def _assert_one_stream_row_bills_both_rounds_once( + logged: tuple[StandardLoggingPayload, ...], client: ResponsesAPIResponse +) -> None: + charged_once: Final = _round_cost(_ROUND_ONE_BODY) + _round_cost(_ROUND_TWO_BODY) - _FIXED_MARGIN + assert len(logged) == 1 + row: Final = logged[0] + assert _output_types(row["response"]["output"]) == _output_types(client.output) + assert "message" in _output_types(client.output) + assert "function_call" not in _output_types(client.output) + assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5) + assert row["cache_hit"] is None + assert row["metadata"]["usage_object"]["prompt_tokens_details"]["cached_tokens"] == 32 + 64 + assert row["hidden_params"]["additional_headers"]["llm_provider-content-type"] == SSE_HEADERS["content-type"] + assert row["response_cost"] == pytest.approx(charged_once) + assert row["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + assert client.usage is not None + assert (client.usage.input_tokens, client.usage.output_tokens) == (71 + 149, 19 + 5) + assert client.usage.cost == pytest.approx(charged_once) + + +def _assert_breakdown_agrees_with_the_billed_cost(row: StandardLoggingPayload) -> None: + breakdown: Final = row.get("cost_breakdown") + assert breakdown is None or breakdown["total_cost"] == pytest.approx(row["response_cost"]) + + +def _request_failure_row(logging_obj: Logging, success_row: StandardLoggingPayload) -> Mapping[str, object]: + failure_row: Final = _failure_fields_to_lift(MappingProxyType({"litellm_logging_obj": logging_obj})) + failure_payload: Final = cast(StandardLoggingPayload, failure_row["standard_logging_object"]) + assert failure_payload["status"] == "failure" + assert failure_payload["id"] != success_row["id"] + return failure_row + + +def _assert_billed_tokens(row: Mapping[str, object], tokens: tuple[int, int]) -> None: + usage: Final = row["combined_usage_object"] + assert isinstance(usage, Usage) + assert (usage.prompt_tokens, usage.completion_tokens) == tokens + + +def _assert_one_row_bills(logged: tuple[StandardLoggingPayload, ...], body: Mapping[str, object]) -> None: + usage: Final = cast(Mapping[str, int], body["usage"]) + assert len(logged) == 1 + assert (logged[0]["prompt_tokens"], logged[0]["completion_tokens"]) == ( + usage["input_tokens"], + usage["output_tokens"], + ) + assert logged[0]["response_cost"] == pytest.approx(_round_cost(body)) + assert logged[0]["response_cost"] > 0 + + +def _assert_one_row_bills_round_one_twice_with_one_fixed_margin(logged: tuple[StandardLoggingPayload, ...]) -> None: + charged_once: Final = 2 * _round_cost(_ROUND_ONE_BODY) - _FIXED_MARGIN + assert len(logged) == 1 + assert (logged[0]["prompt_tokens"], logged[0]["completion_tokens"]) == (2 * 71, 2 * 19) + assert logged[0]["response_cost"] == pytest.approx(charged_once) + assert logged[0]["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_logs_the_final_response_billed_for_both_rounds_with_one_fixed_margin( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY), _streamed(_ROUND_TWO_BODY))) + recorder: Final = _RoundRecorder() + + events: Final = await _stream_auto_execute(_rounds_logging_obj(recorder, "mcp-stream-rounds")) + await _settle(lambda: len(recorder.logged) > 0) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, _final_response(events)) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_deferred_for_a_post_call_guardrail_logs_both_rounds_once_when_the_proxy_releases( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY), _streamed(_ROUND_TWO_BODY))) + recorder: Final = _RoundRecorder() + logging_obj: Final = _rounds_logging_obj(recorder, "mcp-stream-rounds-deferred") + + with _echo_tool_served("PING"): + stream: Final = await _open_stream_auto_execute(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="aresponses", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + events: Final = await _drain(stream) + await asyncio.sleep(0.2) + logged_before_release: Final = recorder.logged + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await _settle(lambda: len(recorder.logged) > 0) + + assert logged_before_release == () + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, _final_response(events)) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bills_the_paid_first_round_on_its_success_row_when_the_follow_up_stream_breaks( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.utils import ProxyLogging + + _serve_responses( + respx_mock, + monkeypatch, + (_streamed(_ROUND_ONE_BODY), httpx.Response(200, stream=_CutAfterTheFirstEvent(), headers=SSE_HEADERS)), + ) + recorder: Final = _RoundRecorder() + logging_obj: Final = _rounds_logging_obj(recorder, "mcp-stream-follow-up-breaks") + + with pytest.raises(litellm.ServiceUnavailableError): + await _stream_auto_execute(logging_obj) + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0) + + _assert_one_row_bills(recorder.logged, _ROUND_ONE_BODY) + assert [row["response_cost"] for row in recorder.failed] == [0.0] + assert _request_failure_row(logging_obj, recorder.logged[0]).get("response_cost", 0.0) == 0.0 + + +@pytest.mark.asyncio +async def test_stream_auto_execute_blocked_by_a_post_call_guardrail_keeps_both_rounds_on_the_success_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY), _streamed(_ROUND_TWO_BODY))) + recorder: Final = _RoundRecorder() + logging_obj: Final = _rounds_logging_obj(recorder, "mcp-stream-rounds-blocked") + blocked: Final = GuardrailRaisedException(guardrail_name="output-policy", message="blocked") + + with _echo_tool_served("PING"): + stream: Final = await _open_stream_auto_execute(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="aresponses", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + events: Final = await _drain(stream) + if not ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, blocked): + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await logging_obj.async_failure_handler(blocked, "blocked by output-policy") + await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, _final_response(events)) + assert [row["response_cost"] for row in recorder.failed] == [0.0] + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bills_the_paid_first_round_once_when_the_follow_up_is_rejected( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + context_window_error: Final = { + "error": { + "message": "Your input exceeds the context window of this model.", + "type": "invalid_request_error", + "param": "input", + "code": "context_length_exceeded", + } + } + _serve_responses( + respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY), httpx.Response(400, json=context_window_error)) + ) + recorder: Final = _RoundRecorder() + + events: Final = await _stream_auto_execute(_rounds_logging_obj(recorder, "mcp-stream-follow-up-rejected")) + await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0) + + assert _event_type(events[-1]) == "error" + _assert_one_row_bills(recorder.logged, _ROUND_ONE_BODY) + assert [row["response_cost"] for row in recorder.failed] == [0.0] + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bills_every_paid_round_with_one_fixed_margin_when_the_third_round_is_rejected( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses( + respx_mock, + monkeypatch, + ( + _streamed(_ROUND_ONE_BODY), + _streamed(_ROUND_ONE_BODY), + httpx.Response(400, json={"error": {"message": "Your input exceeds the context window of this model."}}), + ), + ) + recorder: Final = _RoundRecorder() + + await _stream_auto_execute(_rounds_logging_obj(recorder, "mcp-stream-third-round-rejected")) + await _settle(lambda: len(recorder.logged) > 0) + + _assert_one_row_bills_round_one_twice_with_one_fixed_margin(recorder.logged) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bills_every_finished_round_when_the_client_disconnects_during_a_later_tool_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY), _streamed(_ROUND_ONE_BODY))) + recorder: Final = _RoundRecorder() + logging_obj: Final = _rounds_logging_obj(recorder, "mcp-stream-disconnect-in-tool-call") + echo_then_disconnect: Final = AsyncMock( + side_effect=[[{"tool_call_id": "call_echo", "result": "PING", "name": "echo"}], asyncio.CancelledError()] + ) + + with ( + _echo_tool_served("PING"), + patch.object(LiteLLM_Proxy_MCP_Handler, "execute_tool_calls", echo_then_disconnect), + pytest.raises(asyncio.CancelledError), + ): + await _drain(await _open_stream_auto_execute(logging_obj)) + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await _settle(lambda: len(recorder.logged) > 0) + + _assert_one_row_bills_round_one_twice_with_one_fixed_margin(recorder.logged) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_answered_without_a_tool_call_logs_its_one_round( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_TWO_BODY),)) + recorder: Final = _RoundRecorder() + + events: Final = await _stream_auto_execute(_rounds_logging_obj(recorder, "mcp-stream-one-round")) + await _settle(lambda: len(recorder.logged) > 0) + + assert _output_types(_final_response(events).output) == ("message",) + _assert_one_row_bills(recorder.logged, _ROUND_TWO_BODY) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bills_both_rounds_with_one_fixed_margin_when_the_final_round_is_incomplete( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + truncated: Final = _streamed( + _ROUND_TWO_BODY, status="incomplete", incomplete_details={"reason": "max_output_tokens"} + ) + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY), truncated)) + recorder: Final = _RoundRecorder() + + events: Final = await _stream_auto_execute(_rounds_logging_obj(recorder, "mcp-stream-final-round-incomplete")) + await _settle(lambda: len(recorder.logged) > 0) + + _assert_one_stream_row_bills_both_rounds_once(recorder.logged, _final_response(events, "response.incomplete")) + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bills_a_failed_follow_up_on_its_own_failure_row_with_no_second_fixed_margin( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"openai": {"fixed_amount": _FIXED_MARGIN}}) + server_error: Final = {"code": "server_error", "message": "The server had an error processing your request."} + _serve_responses( + respx_mock, + monkeypatch, + (_streamed(_ROUND_ONE_BODY), _streamed(_ROUND_TWO_BODY, status="failed", error=server_error)), + ) + recorder: Final = _RoundRecorder() + logging_obj: Final = _rounds_logging_obj(recorder, "mcp-stream-follow-up-failed") + + with pytest.raises(litellm.ServiceUnavailableError): + await _stream_auto_execute(logging_obj) + await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0) + + _assert_one_row_bills(recorder.logged, _ROUND_ONE_BODY) + assert len(recorder.failed) == 1 + assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (149, 5) + assert recorder.failed[0]["response_cost"] == pytest.approx(_round_cost(_ROUND_TWO_BODY) - _FIXED_MARGIN) + _assert_breakdown_agrees_with_the_billed_cost(recorder.failed[0]) + failure_row: Final = _request_failure_row(logging_obj, recorder.logged[0]) + _assert_billed_tokens(failure_row, (149, 5)) + assert failure_row["response_cost"] == pytest.approx(_round_cost(_ROUND_TWO_BODY) - _FIXED_MARGIN) + + +_XAI_MODEL: Final = "xai/grok-4.7" +_XAI_RESPONSES_URL: Final = r"https://api\.x\.ai/v1/responses" + + +def _xai_round(body: Mapping[str, object], cost_in_usd_ticks: int) -> Mapping[str, object]: + usage: Final = cast(Mapping[str, object], body["usage"]) + stated: Final = {**usage, "cost_in_usd_ticks": cost_in_usd_ticks, "context_details": {"cached_tokens": 3}} + return MappingProxyType({**body, "model": "grok-4.7", "usage": stated}) + + +def _xai_round_cost(body: Mapping[str, object]) -> float: + usage: Final = cast(Mapping[str, object], body["usage"]) + stated: Final = {**usage, "cost": xai_reported_cost_in_usd(usage["cost_in_usd_ticks"])} + return litellm.completion_cost( + completion_response=ResponsesAPIResponse.model_validate({**body, "usage": stated}), + model=_XAI_MODEL, + custom_llm_provider="xai", + call_type="aresponses", + ) + + +@pytest.mark.asyncio +async def test_xai_stream_auto_execute_bills_each_round_at_its_own_price_with_one_fixed_margin( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"xai": {"fixed_amount": _FIXED_MARGIN}}) + rounds: Final = (_xai_round(_ROUND_ONE_BODY, 12_400_000), _xai_round(_ROUND_TWO_BODY, 8_800_000)) + _serve_responses(respx_mock, monkeypatch, tuple(_streamed(body) for body in rounds), _XAI_RESPONSES_URL) + recorder: Final = _RoundRecorder() + + with _echo_tool_served("PING"): + events: Final = await _drain( + await litellm.aresponses( + model=_XAI_MODEL, + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + api_key="xai-mcp-rounds", + stream=True, + litellm_logging_obj=_rounds_logging_obj(recorder, "mcp-xai-stream-rounds"), + ) + ) + await _settle(lambda: len(recorder.logged) > 0) + + charged_once: Final = _xai_round_cost(rounds[0]) + _xai_round_cost(rounds[1]) - _FIXED_MARGIN + client: Final = _final_response(events) + assert len(recorder.logged) == 1 + assert recorder.logged[0]["response_cost"] == pytest.approx(charged_once) + assert recorder.logged[0]["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + assert client.usage is not None + assert client.usage.cost == pytest.approx(charged_once) + assert json.loads(events[-1].model_dump_json())["response"]["usage"]["context_details"] == {"cached_tokens": 6} + + +_UNPRICED_MODEL: Final = "litellm-unpriced-mcp-rounds" + + +@pytest.mark.asyncio +async def test_stream_auto_execute_on_an_unpriced_model_logs_both_rounds_as_a_failed_cost_lookup_not_a_free_call( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + rounds: Final = tuple( + MappingProxyType({**body, "model": _UNPRICED_MODEL}) for body in (_ROUND_ONE_BODY, _ROUND_TWO_BODY) + ) + _serve_rounds(respx_mock, monkeypatch, rounds) + recorder: Final = _RoundRecorder() + + with _echo_tool_served("PING"): + events: Final = await _drain( + await litellm.aresponses( + model=f"openai/{_UNPRICED_MODEL}", + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + api_key="sk-mcp-rounds", + api_base="https://mcp-rounds.example.com/v1", + stream=True, + litellm_logging_obj=_rounds_logging_obj(recorder, "mcp-stream-unpriced-rounds"), + ) + ) + await _settle(lambda: len(recorder.logged) > 0) + + client: Final = _final_response(events) + assert len(recorder.logged) == 1 + row: Final = recorder.logged[0] + assert (row["prompt_tokens"], row["completion_tokens"]) == (71 + 149, 19 + 5) + assert row["response_cost_failure_debug_info"] is not None + assert client.usage is not None + assert client.usage.cost is None + + +async def _stop_reading_once_the_follow_up_answers(logging_obj: Logging) -> None: + stream: Final = await _open_stream_auto_execute(logging_obj) + async for event in cast(Any, stream): + if _event_type(event) == "response.output_item.added" and _output_types((event.item,)) == ("message",): + return + + +@pytest.mark.asyncio +async def test_stream_auto_execute_bills_the_finished_first_round_when_the_caller_stops_reading_the_follow_up( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY), _streamed(_ROUND_TWO_BODY))) + recorder: Final = _RoundRecorder() + + with _echo_tool_served("PING"): + await _stop_reading_once_the_follow_up_answers(_rounds_logging_obj(recorder, "mcp-stream-caller-stops")) + await _settle(lambda: len(recorder.logged) > 0) + + _assert_one_row_bills(recorder.logged, _ROUND_ONE_BODY) + + +async def _stream_read_until_the_follow_up_answers(logging_obj: Logging) -> object: + stream: Final = await _open_stream_auto_execute(logging_obj) + async for event in cast(Any, stream): + if _event_type(event) == "response.output_item.added" and _output_types((event.item,)) == ("message",): + return stream + pytest.fail("the follow-up round never answered") + + +_STREAM_BILLING_NAMES: Final = ("StreamedRoundBilling", "dispatch_success_handlers", "StreamingIterator") + + +def _names_stream_billing(text: str) -> bool: + return any(name in text for name in _STREAM_BILLING_NAMES) + + +class _UnraisableRecorder: + def __init__(self) -> None: + self.raised: tuple[str, ...] = () + + def __call__(self, unraisable: "sys.UnraisableHookArgs") -> None: + self.raised = (*self.raised, f"{unraisable.object!r}: {unraisable.exc_value!r}") + + +def test_stream_auto_execute_dropped_after_its_event_loop_closed_discards_its_parked_log_cleanly( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_responses(respx_mock, monkeypatch, (_streamed(_ROUND_ONE_BODY), _streamed(_ROUND_TWO_BODY))) + recorder: Final = _RoundRecorder() + unraisable: Final = _UnraisableRecorder() + gc.collect() + monkeypatch.setattr(sys, "unraisablehook", unraisable) + + with _echo_tool_served("PING"), warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + asyncio.run(_stream_read_until_the_follow_up_answers(_rounds_logging_obj(recorder, "mcp-stream-loop-closed"))) + gc.collect() + + assert recorder.logged == () + assert [raised for raised in unraisable.raised if _names_stream_billing(raised)] == [] + assert [str(warning.message) for warning in caught if _names_stream_billing(str(warning.message))] == [] + + +_BRIDGED_MODEL: Final = "anthropic/claude-haiku-5-5" +_ANTHROPIC_MESSAGES_URL: Final = r"https://api\.anthropic\.com/v1/messages" + + +def _bridged_round(content: bytes) -> httpx.Response: + return httpx.Response(200, content=content, headers=SSE_HEADERS) + + +def _bridged_echo_call_round(cache_usage: Mapping[str, object] = MappingProxyType({})) -> httpx.Response: + return _bridged_round( + _anthropic_round( + "msg_echo_call", + {"type": "tool_use", "id": "call_echo", "name": "echo", "input": {}}, + {"type": "input_json_delta", "partial_json": json.dumps({"value": "PING"})}, + "tool_use", + model="claude-haiku-5-5", + usage=(533, 40), + cache_usage=cache_usage, + ) + ) + + +def _bridged_echo_answer_round( + stop: str = "end_turn", cache_usage: Mapping[str, object] = MappingProxyType({}) +) -> httpx.Response: + return _bridged_round( + _anthropic_round( + "msg_echo_answer", + {"type": "text", "text": ""}, + {"type": "text_delta", "text": "PING"}, + stop, + model="claude-haiku-5-5", + usage=(470, 4), + cache_usage=cache_usage, + ) + ) + + +def _anthropic_prompt_cache_usage(written: int, read: int) -> Mapping[str, object]: + return MappingProxyType( + { + "cache_creation_input_tokens": written, + "cache_read_input_tokens": read, + "cache_creation": {"ephemeral_5m_input_tokens": written, "ephemeral_1h_input_tokens": 0}, + "inference_geo": "us", + "speed": "standard", + } + ) + + +class _CutAfterTheMessageStart(httpx.AsyncByteStream): + async def __aiter__(self): + yield _sse((_anthropic_message_start("msg_echo_answer", "claude-haiku-5-5", 470),)) + raise httpx.ReadError("connection reset mid-stream") + + +def _bridged_round_cost(prompt_tokens: int, completion_tokens: int) -> float: + return litellm.completion_cost( + completion_response=ModelResponse( + model="claude-haiku-5-5", + usage=Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ), + ), + model=_BRIDGED_MODEL, + ) + + +async def _open_bridged_stream_auto_execute(logging_obj: Logging) -> object: + return await litellm.aresponses( + model=_BRIDGED_MODEL, + input="Call echo with PING", + tools=[dict(_ECHO_MCP_TOOL)], + api_key="sk-ant-mcp-rounds", + stream=True, + litellm_logging_obj=logging_obj, + ) + + +def _assert_logged_as_the_final_chat_answer_with_both_rounds_usage(recorder: _RoundRecorder, answer: str) -> None: + assert len(recorder.results) == 1 + logged: Final = recorder.results[0] + assert isinstance(logged, ModelResponse) + assert logged.choices[0].message.content == answer + usage: Final[object] = getattr(logged, "usage", None) + assert isinstance(usage, Usage) + assert (usage.prompt_tokens, usage.completion_tokens) == (533 + 470, 40 + 4) + + +def _assert_one_bridged_row_bills_both_rounds_once(recorder: _RoundRecorder, client: ResponsesAPIResponse) -> None: + charged_once: Final = _bridged_round_cost(533, 40) + _bridged_round_cost(470, 4) - _FIXED_MARGIN + assert len(recorder.logged) == 1 + row: Final = recorder.logged[0] + _assert_logged_as_the_final_chat_answer_with_both_rounds_usage(recorder, "PING") + assert row["id"] == client.id + assert "message" in _output_types(client.output) + assert (row["prompt_tokens"], row["completion_tokens"]) == (533 + 470, 40 + 4) + assert row["cache_hit"] is False + assert row["response_cost"] == pytest.approx(charged_once) + assert row["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + assert client.usage is not None + assert (client.usage.input_tokens, client.usage.output_tokens) == (533 + 470, 40 + 4) + assert client.usage.cost == pytest.approx(charged_once) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_logs_the_final_response_billed_for_both_rounds_with_one_fixed_margin( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"anthropic": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses( + respx_mock, monkeypatch, (_bridged_echo_call_round(), _bridged_echo_answer_round()), _ANTHROPIC_MESSAGES_URL + ) + recorder: Final = _RoundRecorder() + + with _echo_tool_served("PING"): + events: Final = await _drain( + await _open_bridged_stream_auto_execute(_rounds_logging_obj(recorder, "mcp-bridged-stream-rounds")) + ) + await _settle(lambda: len(recorder.logged) > 0) + + assert _output_types(_final_response(events).output) == ("mcp_call", "message") + _assert_one_bridged_row_bills_both_rounds_once(recorder, _final_response(events)) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_logs_the_provider_usage_of_both_rounds_summed( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"anthropic": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses( + respx_mock, + monkeypatch, + ( + _bridged_echo_call_round(_anthropic_prompt_cache_usage(written=10, read=5)), + _bridged_echo_answer_round(cache_usage=_anthropic_prompt_cache_usage(written=3, read=7)), + ), + _ANTHROPIC_MESSAGES_URL, + ) + recorder: Final = _RoundRecorder() + + with _echo_tool_served("PING"): + await _drain( + await _open_bridged_stream_auto_execute(_rounds_logging_obj(recorder, "mcp-bridged-provider-usage")) + ) + await _settle(lambda: len(recorder.logged) > 0) + + assert len(recorder.logged) == 1 + row: Final = recorder.logged[0] + usage: Final = row["metadata"]["usage_object"] + assert "choices" in row["response"] + assert (row["prompt_tokens"], row["completion_tokens"]) == (533 + 10 + 5 + 470 + 3 + 7, 40 + 4) + assert (usage["cache_creation_input_tokens"], usage["cache_read_input_tokens"]) == (10 + 3, 5 + 7) + assert usage["prompt_tokens_details"]["cache_creation_token_details"]["ephemeral_5m_input_tokens"] == 10 + 3 + assert (usage["inference_geo"], usage["speed"]) == ("us", "standard") + assert usage["cost"] == pytest.approx(row["response_cost"]) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_bills_both_rounds_with_one_fixed_margin_when_the_final_round_is_incomplete( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"anthropic": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses( + respx_mock, + monkeypatch, + (_bridged_echo_call_round(), _bridged_echo_answer_round("max_tokens")), + _ANTHROPIC_MESSAGES_URL, + ) + recorder: Final = _RoundRecorder() + + with _echo_tool_served("PING"): + events: Final = await _drain( + await _open_bridged_stream_auto_execute(_rounds_logging_obj(recorder, "mcp-bridged-stream-incomplete")) + ) + await _settle(lambda: len(recorder.logged) > 0) + + _assert_one_bridged_row_bills_both_rounds_once(recorder, _final_response(events, "response.incomplete")) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_deferred_for_a_post_call_guardrail_logs_both_rounds_once_on_release( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"anthropic": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses( + respx_mock, monkeypatch, (_bridged_echo_call_round(), _bridged_echo_answer_round()), _ANTHROPIC_MESSAGES_URL + ) + recorder: Final = _RoundRecorder() + logging_obj: Final = _rounds_logging_obj(recorder, "mcp-bridged-stream-rounds-deferred") + + with _echo_tool_served("PING"): + stream: Final = await _open_bridged_stream_auto_execute(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="aresponses", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + events: Final = await _drain(stream) + await asyncio.sleep(0.2) + logged_before_release: Final = recorder.logged + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await _settle(lambda: len(recorder.logged) > 0) + + assert logged_before_release == () + _assert_one_bridged_row_bills_both_rounds_once(recorder, _final_response(events)) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_blocked_by_a_post_call_guardrail_bills_both_rounds_on_the_failure_row( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.proxy.utils import ProxyLogging + + monkeypatch.setattr(litellm, "cost_margin_config", {"anthropic": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses( + respx_mock, monkeypatch, (_bridged_echo_call_round(), _bridged_echo_answer_round()), _ANTHROPIC_MESSAGES_URL + ) + recorder: Final = _RoundRecorder() + logging_obj: Final = _rounds_logging_obj(recorder, "mcp-bridged-stream-rounds-blocked") + blocked: Final = GuardrailRaisedException(guardrail_name="output-policy", message="blocked") + + with _echo_tool_served("PING"): + stream: Final = await _open_bridged_stream_auto_execute(logging_obj) + ProxyBaseLLMRequestProcessing(data={})._arm_deferred_stream_dispatch( + response=stream, route_type="aresponses", user_api_key_dict=MagicMock(), logging_obj=logging_obj + ) + await _drain(stream) + ProxyLogging._discard_deferred_stream_logging_for_failure({"litellm_logging_obj": logging_obj}, blocked) + ProxyLogging.fire_deferred_stream_logging({"litellm_logging_obj": logging_obj}) + await logging_obj.async_failure_handler(blocked, "blocked by output-policy") + await _settle(lambda: len(recorder.failed) > 0) + + charged_once: Final = _bridged_round_cost(533, 40) + _bridged_round_cost(470, 4) - _FIXED_MARGIN + assert recorder.logged == () + assert len(recorder.failed) == 1 + assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (533 + 470, 40 + 4) + assert recorder.failed[0]["response_cost"] == pytest.approx(charged_once) + assert recorder.failed[0]["cost_breakdown"]["total_cost"] == pytest.approx(charged_once) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stop", ("end_turn", "max_tokens"), ids=("completed", "incomplete")) +async def test_bridged_stream_auto_execute_bills_both_rounds_when_the_caller_stops_reading_after_the_answer_text( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, stop: str +): + _serve_responses( + respx_mock, monkeypatch, (_bridged_echo_call_round(), _bridged_echo_answer_round(stop)), _ANTHROPIC_MESSAGES_URL + ) + recorder: Final = _RoundRecorder() + + async def stop_reading_once_the_answer_text_is_done() -> None: + stream: Final = await _open_bridged_stream_auto_execute( + _rounds_logging_obj(recorder, "mcp-bridged-stream-caller-stops") + ) + async for event in cast(Any, stream): + if _event_type(event) == "response.output_text.done": + return + + with _echo_tool_served("PING"): + await stop_reading_once_the_answer_text_is_done() + await _settle(lambda: len(recorder.logged) > 0) + + assert len(recorder.logged) == 1 + _assert_logged_as_the_final_chat_answer_with_both_rounds_usage(recorder, "PING") + assert (recorder.logged[0]["prompt_tokens"], recorder.logged[0]["completion_tokens"]) == (533 + 470, 40 + 4) + assert recorder.logged[0]["response_cost"] == pytest.approx( + _bridged_round_cost(533, 40) + _bridged_round_cost(470, 4) + ) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_bills_a_broken_follow_up_on_its_own_failure_row_with_no_second_fixed_margin( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"anthropic": {"fixed_amount": _FIXED_MARGIN}}) + _serve_responses( + respx_mock, + monkeypatch, + (_bridged_echo_call_round(), httpx.Response(200, stream=_CutAfterTheMessageStart(), headers=SSE_HEADERS)), + _ANTHROPIC_MESSAGES_URL, + ) + recorder: Final = _RoundRecorder() + logging_obj: Final = _rounds_logging_obj(recorder, "mcp-bridged-stream-breaks") + + with _echo_tool_served("PING"), pytest.raises(MidStreamFallbackError): + await _drain(await _open_bridged_stream_auto_execute(logging_obj)) + await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0) + + assert len(recorder.logged) == 1 + assert (recorder.logged[0]["prompt_tokens"], recorder.logged[0]["completion_tokens"]) == (533, 40) + assert recorder.logged[0]["response_cost"] == pytest.approx(_bridged_round_cost(533, 40)) + assert len(recorder.failed) == 1 + assert (recorder.failed[0]["prompt_tokens"], recorder.failed[0]["completion_tokens"]) == (470, 0) + assert recorder.failed[0]["response_cost"] == pytest.approx(_bridged_round_cost(470, 0) - _FIXED_MARGIN) + _assert_breakdown_agrees_with_the_billed_cost(recorder.failed[0]) + failure_row: Final = _request_failure_row(logging_obj, recorder.logged[0]) + _assert_billed_tokens(failure_row, (470, 0)) + assert failure_row["response_cost"] == pytest.approx(_bridged_round_cost(470, 0) - _FIXED_MARGIN) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_bills_a_broken_follow_up_its_own_fixed_margin_after_a_cached_first_round( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cost_margin_config", {"anthropic": {"fixed_amount": _FIXED_MARGIN}}) + monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local")) + _serve_responses( + respx_mock, + monkeypatch, + ( + _bridged_echo_call_round(), + _bridged_echo_answer_round(), + httpx.Response(200, stream=_CutAfterTheMessageStart(), headers=SSE_HEADERS), + ), + _ANTHROPIC_MESSAGES_URL, + ) + warm: Final = _RoundRecorder() + with _echo_tool_served("PING"): + await _drain( + await _open_bridged_stream_auto_execute(_rounds_logging_obj(warm, "mcp-bridged-cache-then-break-warm")) + ) + await _settle(lambda: len(warm.logged) > 0) + + recorder: Final = _RoundRecorder() + with _echo_tool_served("PONG"), pytest.raises(MidStreamFallbackError): + await _drain( + await _open_bridged_stream_auto_execute(_rounds_logging_obj(recorder, "mcp-bridged-cache-then-break")) + ) + await _settle(lambda: len(recorder.logged) > 0 and len(recorder.failed) > 0) + + assert [row["response_cost"] for row in recorder.logged] == [0.0] + assert len(recorder.failed) == 1 + assert recorder.failed[0]["response_cost"] == pytest.approx(_bridged_round_cost(470, 0)) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_answered_without_a_tool_call_logs_its_one_round( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + _serve_responses(respx_mock, monkeypatch, (_bridged_echo_answer_round(),), _ANTHROPIC_MESSAGES_URL) + recorder: Final = _RoundRecorder() + + with _echo_tool_served("PING"): + await _drain(await _open_bridged_stream_auto_execute(_rounds_logging_obj(recorder, "mcp-bridged-one-round"))) + await _settle(lambda: len(recorder.logged) > 0) + + assert len(recorder.logged) == 1 + assert "choices" in recorder.logged[0]["response"] + assert (recorder.logged[0]["prompt_tokens"], recorder.logged[0]["completion_tokens"]) == (470, 4) + assert recorder.logged[0]["response_cost"] == pytest.approx(_bridged_round_cost(470, 4)) + + +@pytest.mark.asyncio +async def test_bridged_stream_auto_execute_bills_nothing_for_a_round_served_from_the_response_cache( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "cache", litellm.Cache(type="local")) + provider: Final = _serve_responses( + respx_mock, + monkeypatch, + (_bridged_echo_call_round(), _bridged_echo_answer_round(), _bridged_echo_answer_round()), + _ANTHROPIC_MESSAGES_URL, + ) + warm: Final = _RoundRecorder() + with _echo_tool_served("PING"): + await _drain(await _open_bridged_stream_auto_execute(_rounds_logging_obj(warm, "mcp-bridged-cache-warm"))) + await _settle(lambda: len(warm.logged) > 0) + + first_cached: Final = _RoundRecorder() + with _echo_tool_served("PONG"): + events: Final = await _drain( + await _open_bridged_stream_auto_execute(_rounds_logging_obj(first_cached, "mcp-bridged-cache-first-round")) + ) + await _settle(lambda: len(first_cached.logged) > 0) + + both_cached: Final = _RoundRecorder() + with _echo_tool_served("PING"): + await _drain( + await _open_bridged_stream_auto_execute(_rounds_logging_obj(both_cached, "mcp-bridged-cache-both-rounds")) + ) + await _settle(lambda: len(both_cached.logged) > 0) + + client: Final = _final_response(events) + assert provider.call_count == 3 + assert len(first_cached.logged) == 1 + assert (first_cached.logged[0]["prompt_tokens"], first_cached.logged[0]["completion_tokens"]) == (533 + 470, 40 + 4) + assert first_cached.logged[0]["response_cost"] == pytest.approx(_bridged_round_cost(470, 4)) + assert first_cached.logged[0]["cache_hit"] is False + assert [(row["response_cost"], row["cache_hit"]) for row in both_cached.logged] == [(0.0, True)] + assert client.usage is not None + assert client.usage.cost == pytest.approx(_bridged_round_cost(470, 4)) diff --git a/tests/unit/responses/test_streaming_iterator_error_events.py b/tests/unit/responses/test_streaming_iterator_error_events.py index 04d2b9ddf81..7445070bd3a 100644 --- a/tests/unit/responses/test_streaming_iterator_error_events.py +++ b/tests/unit/responses/test_streaming_iterator_error_events.py @@ -23,7 +23,8 @@ import asyncio import json from datetime import datetime from importlib import import_module -from typing import Final +from collections.abc import Mapping +from typing import Final, cast from unittest.mock import Mock, patch import httpx @@ -31,6 +32,7 @@ import pytest import litellm from litellm.exceptions import MidStreamFallbackError +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.get_litellm_params import get_litellm_params from litellm.litellm_core_utils.litellm_logging import _get_provider_request_id @@ -44,6 +46,7 @@ from litellm.responses.streaming_iterator import ( _status_code_for_error_fields, ) from litellm.utils import ProviderConfigManager +from litellm.types.utils import StandardLoggingPayload from litellm.types.llms.openai import ( ErrorEvent, ErrorEventError, @@ -523,6 +526,82 @@ def test_handle_logging_failed_response_records_usage_and_cost(): iterator.logging_obj.response_cost_calculator.assert_called_once_with(result=chunk.response) +class _FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failed: tuple[StandardLoggingPayload, ...] = () + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.failed = (*self.failed, cast(StandardLoggingPayload, kwargs["standard_logging_object"])) + + +async def _first_failure_row(recorder: _FailureRecorder) -> StandardLoggingPayload: + for _ in range(200): + if recorder.failed: + return recorder.failed[0] + await asyncio.sleep(0.01) + pytest.fail("the failed stream logged no failure row") + + +@pytest.mark.asyncio +async def test_failed_stream_bills_only_its_own_usage_over_an_earlier_attempts_stash(): + recorder: Final = _FailureRecorder() + logging_obj: Final = LiteLLMLoggingObj( + model="gpt-5.6", + messages=[], + stream=True, + call_type="aresponses", + start_time=datetime(2026, 1, 1), + litellm_call_id="responses-failed-over-stash", + function_id="responses-failed-over-stash", + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="gpt-5.6", optional_params={}, litellm_params=get_litellm_params(custom_llm_provider="openai") + ) + logging_obj.model_call_details["combined_usage_object"] = litellm.Usage( + prompt_tokens=100, completion_tokens=50, total_tokens=150 + ) + logging_obj.model_call_details["response_cost"] = 0.5 + failed: Final = { + "type": "response.failed", + "sequence_number": 1, + "response": { + "id": "resp_failed_over_stash", + "object": "response", + "created_at": 1790000000, + "status": "failed", + "model": "gpt-5.6", + "output": [], + "error": {"code": "server_error", "message": "boom"}, + "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15}, + }, + } + iterator: Final = ResponsesAPIStreamingIterator( + response=httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=f"data: {json.dumps(failed)}\n\n".encode() + ), + model="gpt-5.6", + responses_api_provider_config=ProviderConfigManager.get_provider_responses_api_config( + provider="openai", model="gpt-5.6" + ), + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + + with pytest.raises(MidStreamFallbackError): + await anext(aiter(iterator)) + + row: Final = await _first_failure_row(recorder) + prompt_cost, completion_cost = litellm.cost_per_token( + model="gpt-5.6", prompt_tokens=10, completion_tokens=5, custom_llm_provider="openai" + ) + assert (row["prompt_tokens"], row["completion_tokens"]) == (10, 5) + assert row["response_cost"] == pytest.approx(prompt_cost + completion_cost) + + def test_handle_logging_failed_response_without_usage_skips_recording(): iterator = _make_iterator() iterator.completed_response = _make_failed_chunk( From 8fe0a7358e87cad3516d005854b858b14c32f515 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:23:30 +0000 Subject: [PATCH 048/235] ci(integration): keep the MCP SDK1 peer on mcp 1.x and autovacuum off the size cleanup test (#47651) * test(integration): keep autovacuum off the size cleanup test's scratch tables Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * ci(circleci): keep the MCP SDK1 peer venv on mcp 1.x despite the project override Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/config.yml | 2 +- tests/integration/database/test_cleanup_transactions.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d442e32b0a..1edf22578ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1291,7 +1291,7 @@ jobs: name: Install MCP SDK1 peer command: | uv venv --python 3.12 .venv-mcp-peer - uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' + uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1' echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV" - when: condition: diff --git a/tests/integration/database/test_cleanup_transactions.py b/tests/integration/database/test_cleanup_transactions.py index 72da2bb48ee..202657d180e 100644 --- a/tests/integration/database/test_cleanup_transactions.py +++ b/tests/integration/database/test_cleanup_transactions.py @@ -113,7 +113,7 @@ async def test_size_rule_deletes_the_oldest_rows_until_under_the_target_and_does """ Deleting rows never shrinks the table file, so the size pass must stop on live bytes, not on pg_total_relation_size, taking the oldest rows first and keeping the tool index on the same cutoff. Once - autovacuum has turned the deleted rows into free space the catalog no longer shows them as dead, and the + VACUUM has turned the deleted rows into free space the catalog no longer shows them as dead, and the files still measure the same; the next check must read what the last run freed from its saved count and leave the table alone instead of calling the whole file live again. """ @@ -140,13 +140,15 @@ async def test_size_rule_deletes_the_oldest_rows_until_under_the_target_and_does setup.execute( sql.SQL( 'CREATE TABLE {} (request_id text, "startTime" timestamp NOT NULL, payload text, ' - 'PRIMARY KEY (request_id, "startTime"))' + 'PRIMARY KEY (request_id, "startTime")) ' + 'WITH (autovacuum_enabled = false, toast.autovacuum_enabled = false)' ).format(logs) ) setup.execute( sql.SQL( "CREATE TABLE {} (request_id text, tool_name text, start_time timestamp NOT NULL, " - "PRIMARY KEY (request_id, tool_name))" + "PRIMARY KEY (request_id, tool_name)) " + "WITH (autovacuum_enabled = false, toast.autovacuum_enabled = false)" ).format(index) ) setup.execute(sql.SQL("CREATE TABLE {} (param_name text PRIMARY KEY, param_value jsonb)").format(config)) From 8d6b8dab3db519d33cdb7b94b4dba37f1269cde4 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 03:24:59 +0000 Subject: [PATCH 049/235] fix(cli): send pi's session id from lite pi so sessions group and stick (#47700) * fix(cli): send pi's session id from lite pi so sessions group and stick * docs(cli): drop the provider_block docstring lines the README already covers * docs(cli): note the session id format the proxy reads for lite pi --- litellm/proxy/client/cli/README.md | 2 +- litellm/proxy/client/cli/commands/pi.py | 4 +++- tests/unit/proxy/client/cli/test_pi.py | 7 ++++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 842705ad5a7..4038ff6f582 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -494,7 +494,7 @@ Each command resolves your LiteLLM key (logging in via SSO when none is stored a The right variables are picked per agent. Claude Code gets `ANTHROPIC_BASE_URL` (the proxy root, so it appends `/v1/messages`) and `ANTHROPIC_AUTH_TOKEN`, with any stray `ANTHROPIC_API_KEY` cleared so the proxy token wins, and `ENABLE_TOOL_SEARCH=true` (unless you already set it) so Claude Code keeps tool search on even though the base URL is a proxy rather than a first-party Anthropic host. It also gets `CLAUDE_CODE_ENABLE_GATEWAY_MODEL_DISCOVERY=1` (again unless you already set it) so Claude Code v2.1.129+ fills its `/model` picker from the proxy's `/v1/models`; Claude Code only lists entries whose id contains `claude` or `anthropic`, so the proxy lists every other group to Claude Code as `claude-router-` and marks a group whose input window reaches 1M with `[1m]`, and a request on such an id is served by the group. Older Claude Code versions ignore the variable. Export it as `0` to turn discovery off. Codex and OpenCode get `OPENAI_BASE_URL` (the proxy plus `/v1`) and `OPENAI_API_KEY`. Codex ignores `OPENAI_BASE_URL`, so it is additionally pointed at the proxy through a custom provider passed as `-c` config overrides (HTTP/SSE Responses transport, since the proxy does not speak the Responses WebSocket protocol). OpenCode additionally gets `OPENCODE_CONFIG_CONTENT` holding a generated `litellm` provider (`@ai-sdk/openai-compatible`, the proxy `/v1` URL, `{env:OPENAI_API_KEY}`) with one model entry per chat model your key can see on `/v1/models`, so its model picker mirrors the proxy without a hand-maintained `opencode.json`; OpenCode merges that over your own config files, and if you already export `OPENCODE_CONFIG_CONTENT` yours is left alone. When the list cannot be fetched, `lite opencode` says so on stderr and launches anyway. Codex gets the same list as a catalog file, `$CODEX_HOME/litellm-models.json` (default `~/.codex/`), passed as `-c model_catalog_json=` so `/model` lists exactly the proxy's chat models; a proxy model the installed Codex already knows (`gpt-5.5`, say) keeps that Codex's own entry, reasoning levels and prompt included, and only its place in the picker comes from the proxy, while a model Codex does not know gets the plain entry Codex uses for an unknown `-m` slug. Before launching, `lite codex` asks the installed Codex for its own list and then has it read the written file back (both through `codex debug models`), and when the fetch, either of those or the write fails (Codex releases older than 0.130 have no such command) it says so on stderr and launches with Codex's built-in catalog, leaving a rejected file in place. -pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. Your own flags come after the injected pin, so `lite pi --model litellm/` wins, and inside the TUI the `/model` picker lists every synced litellm model. +pi ignores base-URL environment variables entirely, so `lite pi` (kept out of the `lite --help` command listing for now, but fully functional) wires it up differently: before handoff it fetches the models your key can use from the proxy's `/v1/models` (plus each model's context window and output cap from `/model_group/info`, when available) and syncs them into a `litellm` provider entry in pi's `~/.pi/agent/models.json` (honoring `PI_CODING_AGENT_DIR`), then starts pi on that provider's first model via an injected `--model litellm/`. Only that one provider entry is rewritten; the rest of the file, including any other custom providers, is left alone. The entry references the key as `$LITELLM_PROXY_API_KEY`, which the wrapper exports for the session, so the token itself never lands on disk and plain `pi` outside the wrapper simply shows the litellm models as unavailable. The entry also turns on pi's session header (`x-session-id`, sent by pi 0.80.7 and later), so every chat turn of one pi session shares one session id in the logs and sticks to one deployment when session affinity is on. A custom `pi --session-id` must be at least 8 letters, digits, `_`, or `-` for the proxy to read it. Your own flags come after the injected pin, so `lite pi --model litellm/` wins, and inside the TUI the `/model` picker lists every synced litellm model. Options (these belong to the wrapper, so put them before the agent's own flags): diff --git a/litellm/proxy/client/cli/commands/pi.py b/litellm/proxy/client/cli/commands/pi.py index b228617699c..357b82bdf67 100644 --- a/litellm/proxy/client/cli/commands/pi.py +++ b/litellm/proxy/client/cli/commands/pi.py @@ -191,7 +191,7 @@ def provider_block( Real contextWindow/maxTokens matter: pi otherwise assumes 128k/16384, which breaks compaction thresholds and over-asks models with smaller output caps. pi sniffs compat from the base URL, and one gateway URL fronts models with - different capabilities, so both flags are pinned off. + different capabilities, so the store and long-cache flags are pinned off. """ return { "baseUrl": base_url.rstrip("/") + "/v1", @@ -199,6 +199,8 @@ def provider_block( "compat": { "supportsStore": False, "supportsLongCacheRetention": False, + "sendSessionAffinityHeaders": True, + "sessionAffinityFormat": "openrouter", }, "apiKey": f"${LITELLM_PROXY_API_KEY_ENV}", "models": [_model_entry(model_id, limits) for model_id in model_ids], diff --git a/tests/unit/proxy/client/cli/test_pi.py b/tests/unit/proxy/client/cli/test_pi.py index ff4dc25cd0c..e6669564358 100644 --- a/tests/unit/proxy/client/cli/test_pi.py +++ b/tests/unit/proxy/client/cli/test_pi.py @@ -195,7 +195,12 @@ class TestProviderBlock: assert block == { "baseUrl": "http://localhost:4000/v1", "api": "openai-completions", - "compat": {"supportsStore": False, "supportsLongCacheRetention": False}, + "compat": { + "supportsStore": False, + "supportsLongCacheRetention": False, + "sendSessionAffinityHeaders": True, + "sessionAffinityFormat": "openrouter", + }, "apiKey": "$LITELLM_PROXY_API_KEY", "models": [{"id": "m-1"}, {"id": "m-2"}], } From 4ecc5fd13ab80e88d2c9de569cdeab3114e3cfd1 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 20:44:41 -0700 Subject: [PATCH 050/235] feat(azure): add azure_ai/MAI-Code-1.1-Flash (#47779) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 28 +++++++++++++++++++ model_prices_and_context_window.json | 28 +++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 3d89af9d691..fd72d209ffb 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -77018,5 +77018,33 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": false + }, + "azure_ai/MAI-Code-1.1-Flash": { + "cache_read_input_token_cost": 2e-08, + "deprecation_date": "2026-12-09", + "input_cost_per_token": 2e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 256000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true } } diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 3d89af9d691..fd72d209ffb 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -77018,5 +77018,33 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": false + }, + "azure_ai/MAI-Code-1.1-Flash": { + "cache_read_input_token_cost": 2e-08, + "deprecation_date": "2026-12-09", + "input_cost_per_token": 2e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 256000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 1.2e-06, + "source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/responses" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true } } From 045fb7ba84803e3265462c7988ecfaab35351860 Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Tue, 20 Oct 2026 23:18:42 -0500 Subject: [PATCH 051/235] feat(vertex-ai)!: support gemini 3.1 flash tts (#31915) * feat(vertex-ai): support gemini 3.1 flash tts * fix(vertex-ai): keep TTS metadata compatible with the cost map guard * fix(vertex-ai): preserve existing Gemini TTS models and voice mappings * fix(vertex-ai): recognize provider-stripped TTS model names * test(vertex-ai): match current speech HTTP error response * fix(vertex-ai): preserve nested speech locale and voice * fix(vertex-ai): use provider speech media type in proxy * fix(gemini): preserve TTS dispatch and nested speech settings * test(gemini): run TTS regressions in CI shards * fix(vertex-ai): account for Gemini Cloud TTS audio cost * fix(vertex-ai): bill Gemini TTS without optional audio decoder * test(gemini): cover TTS mapping and billing edge cases * test(vertex-ai): cover Gemini voice normalization in provider shard * fix(vertex-ai): keep Gemini TTS routing with generic provider metadata * chore(vertex-ai): remove obsolete lint suppressions * fix(gemini): strip audio format from speech configuration * chore(vertex-ai): remove obsolete mutation suppressions * fix(vertex-ai): preserve FLAC billing and wrapped audio headers * fix(vertex-ai): reject unmeasurable FLAC before billing * fix(vertex-ai): reconcile speech support with current provider mappings * fix(vertex-ai): validate speech mappings and usage types * fix(vertex-ai): preserve speech mapper override forwarding * fix(gemini): map voice aliases across speech request forms * fix(vertex-ai): send only formats the speech bridge cannot serve to Cloud TTS Gemini TTS detection now goes by model name, so newer Vertex Gemini TTS models keep working, and only mp3, opus, alaw, mulaw and other container formats move to Cloud Text-to-Speech. wav and pcm requests stay on generateContent as on main. A Cloud TTS response whose duration cannot be measured now returns the audio unbilled with a warning instead of raising after a paid call. FLAC handling is removed because Cloud TTS has no FLAC encoding. The cost map, the google_genai mappers and the proxy error path go back to main, since those changes were outside this feature. * fix(vertex_ai): keep Gemini TTS formats, region and instructions on routes that serve them Only mp3, opus, alaw and mulaw go to Cloud Text-to-Speech. Every other format gets a local 400 that lists the supported ones, before Google is called. Cloud TTS uses the deployment's us or eu regional host. Other regions and a custom api_base get a 400 that says to request wav or pcm. instructions is sent as the Cloud TTS style prompt, and a request with no voice keeps the Gemini model. get_usage_object is back to main's code, so dict usages still go through Usage.__init__, and the proxy no longer labels pcm16 or linear16 output as WAV. * fix(vertex_ai): send Gemini TTS to every regional Cloud TTS host and keep speak tags as text Cloud Text-to-Speech serves Gemini TTS on a host named after each Vertex location, so a non-global vertex_location now uses {location}-texttospeech.googleapis.com instead of only the us and eu hosts. Google answers for regions it does not serve. Gemini TTS voices reject SSML, so input that contains is sent as text for Gemini TTS models. Chirp keeps the SSML auto-detect. Removes the proxy alaw and mulaw media-type entries. Cloud TTS returns those encodings with a WAV header, so the response is labeled audio/wav and the entries never applied. Adds a regression test that custom per-token pricing bills the usage a speech response reports. * fix(vertex_ai): map Gemini TTS regions to Google's Cloud TTS locations and keep a bad voice from cooling down the deployment US and EU Vertex regions go to the us and eu Cloud TTS hosts, northamerica-northeast1 to its own, global or unset to global, and any other region gets a local 400. A Cloud TTS 404 on the Gemini path, such as a misspelled voice, comes back as a 400 so the router keeps the deployment serving wav. Usage reads the sent request from the response instead of the logging object. * fix(vertex_ai): default Gemini TTS Cloud formats to a Gemini voice and validate speakers Parse only usage fields from the sent Cloud TTS request, so a numeric speakingRate no longer fails after Google bills it. Send the Gemini voice Kore when no voice is named, and reject Cloud TTS voice names unless drop_params is set. Reject malformed speaker entries unless drop_params is set, and translate Cloud TTS speaker fields on wav and chat audio. Use the Vertex default region when no location is set, so mp3 and wav share a region. * test(vertex_ai): cover two-speaker Gemini TTS mp3 through aspeech with fake credentials Replaces two tests that patched the Vertex TTS config's own token and URL methods. * fix(vertex_ai): reject malformed Gemini TTS voices and keep beta Chirp and Lyria params unmapped * fix(vertex_ai): drop unknown Gemini TTS voice keys on wav and pcm instead of rejecting them * fix(vertex_ai): reject unknown Gemini TTS voice keys on wav, pcm, and chat audio unless drop_params is set * fix(vertex_ai): honor every drop_params setting in the Gemini TTS voice key check Global drop_params, batch row and deployment drop_params on Vertex batch file uploads now drop unknown voice keys. A voice object with both speechConfig and speech_config gets a 400 for the ignored speech_config. * fix(gemini): drop unknown nested voice fields under drop_params and detect TTS by served model * fix(gemini): filter unknown speech config fields with fixed-depth helpers and pass audio params through unchanged * fix(gemini): camel-case snake replicated voice keys so drop_params keeps them * fix(vertex-ai): reject voice fields Cloud TTS cannot send unless drop_params is set * fix(vertex-ai): reject wrong-typed Cloud TTS voice values and check speakers as sent Gemini TTS on mp3, opus, alaw, or mulaw now answers 400 for a voice field whose value Cloud TTS cannot send, such as a list languageCode or a string voiceConfig, and for nested voice fields under a speaker that also names speakerId. drop_params still drops them. * fix(vertex-ai): reject wrong-typed speech config wrappers and speakerId overriding a prebuilt voice on Cloud TTS * fix(vertex-ai): keep speakers and voice_config beside a speech_config wrapper on Cloud TTS * fix(vertex-ai): pick Cloud TTS voices from the same speech config the WAV path sends to Gemini * fix(files): carry the deployment's drop_params on model= file uploads so Vertex batch rows drop unknown voice keys * fix(gemini): keep a structured speech voice when the model is a model_alias_map alias for a Gemini TTS model --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/cost_calculator.py | 6 +- .../speech_to_completion_bridge/handler.py | 1 + .../transformation.py | 67 +- .../llms/vertex_ai/files/transformation.py | 8 +- .../vertex_and_google_ai_studio_gemini.py | 334 ++++- .../text_to_speech/transformation.py | 567 +++++++- litellm/main.py | 7 +- .../openai_files_endpoints/common_utils.py | 12 + .../openai_files_endpoints/files_endpoints.py | 3 +- litellm/types/llms/vertex_ai.py | 10 + .../types/llms/vertex_ai_text_to_speech.py | 17 +- litellm/utils.py | 5 + tests/audio_tests/gemini_tts_speech.mp3 | Bin 0 -> 4992 bytes tests/audio_tests/gemini_tts_speech.ogg | Bin 0 -> 4179 bytes .../test_transformation.py | 121 +- tests/unit/llms/gemini/test_gemini_tts.py | 262 +++- .../files/test_vertex_ai_files_streaming.py | 40 + ...test_vertex_and_google_ai_studio_gemini.py | 239 ++++ .../text_to_speech/test_transformation.py | 1233 ++++++++++++++++- .../test_files_endpoint.py | 43 + tests/unit/test_cost_calculator.py | 29 +- 21 files changed, 2878 insertions(+), 126 deletions(-) create mode 100644 tests/audio_tests/gemini_tts_speech.mp3 create mode 100644 tests/audio_tests/gemini_tts_speech.ogg diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index f952883bc87..550bbfcde98 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -1673,8 +1673,10 @@ def completion_cost( verbose_logger.debug("selected model name for cost calculation: %s", model) if completion_response is not None and ( - isinstance(completion_response, BaseModel) or isinstance(completion_response, dict) - ): # tts returns a custom class + isinstance(completion_response, (BaseModel, dict)) + or isinstance(completion_response, HttpxBinaryResponseContent) + and completion_response.usage is not None + ): if isinstance(completion_response, dict): usage_obj: dict | Usage | None = completion_response.get("usage", {}) else: diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py index 00d112773f1..faf2daecf5f 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/handler.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/handler.py @@ -122,6 +122,7 @@ class SpeechToCompletionBridgeHandler: if isinstance(result, ModelResponse): return self.transformation_handler.transform_response( model_response=result, + model=model, response_format=requested_response_format if isinstance(requested_response_format, str) else None, ) else: diff --git a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py index 9c1e5ae7a29..ad6ed269adf 100644 --- a/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py +++ b/litellm/endpoints/speech/speech_to_completion_bridge/transformation.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, cast from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param if TYPE_CHECKING: @@ -38,6 +39,11 @@ class ChatAudioParam(TypedDict): format: ReadOnly[NotRequired[str]] +class ChatSpeechConfigAudioParam(TypedDict): + speech_config: ReadOnly[Mapping[str, object]] + format: ReadOnly[str] + + class SpeechToCompletionBridgeTransformationHandler: def _validate_response_format( self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object] @@ -49,6 +55,20 @@ class SpeechToCompletionBridgeTransformationHandler: return from litellm.exceptions import BadRequestError + if custom_llm_provider.startswith("vertex_ai"): + from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig + + vertex_supported: Final = ", ".join( + sorted({*GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS, *VertexAITextToSpeechConfig.GEMINI_FORMAT_MAPPINGS}) + ) + raise BadRequestError( + message=( + f"Vertex AI Gemini TTS does not support response_format='{response_format}'." + f" Supported response formats: {vertex_supported}." + ), + model=model, + llm_provider=custom_llm_provider, + ) supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS)) raise BadRequestError( message=( @@ -75,8 +95,27 @@ class SpeechToCompletionBridgeTransformationHandler: return response_format if isinstance(response_format, str) else None def _chat_audio_param( - self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object] - ) -> ChatAudioParam | None: + self, + model: str, + voice: str | Mapping[str, object] | None, + optional_params: Mapping[str, object], + drop_params: bool, + custom_llm_provider: str, + ) -> ChatAudioParam | ChatSpeechConfigAudioParam | None: + import litellm + + if isinstance(voice, Mapping) and self._is_gemini_tts_model(litellm.model_alias_map.get(model, model)): + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + normalize_gemini_speech_config, + reject_unknown_gemini_tts_voice_keys, + ) + + reject_unknown_gemini_tts_voice_keys(model, voice, drop_params, custom_llm_provider) + speech_audio: Final[ChatSpeechConfigAudioParam] = { + "speech_config": normalize_gemini_speech_config(voice, drops_unsupported_params(drop_params)), + "format": GEMINI_TTS_CHAT_AUDIO_FORMAT, + } + return speech_audio if not isinstance(voice, str): return None audio_format: Final = self._chat_audio_format(model, optional_params) @@ -90,9 +129,9 @@ class SpeechToCompletionBridgeTransformationHandler: self, model: str, input: str, - voice: str | dict | None, - optional_params: dict, - litellm_params: dict, + voice: str | Mapping[str, object] | None, + optional_params: Mapping[str, object], + litellm_params: Mapping[str, object], headers: dict, litellm_logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str, @@ -104,7 +143,13 @@ class SpeechToCompletionBridgeTransformationHandler: "messages": [user_message], "modalities": ["audio"], **self._chat_completion_params(optional_params), - "audio": self._chat_audio_param(model, voice, optional_params), + "audio": self._chat_audio_param( + model, + voice, + optional_params, + drops_unsupported_params(litellm_params.get("drop_params")), + custom_llm_provider, + ), **litellm_params, "headers": headers, "litellm_logging_obj": litellm_logging_obj, @@ -153,8 +198,9 @@ class SpeechToCompletionBridgeTransformationHandler: return wav_header + pcm_data def _is_gemini_tts_model(self, model: str) -> bool: - """Check if the model is a Gemini TTS model that returns PCM16 data.""" - return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower()) + from litellm.utils import is_gemini_tts_model + + return is_gemini_tts_model(model) def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]: if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT: @@ -162,7 +208,7 @@ class SpeechToCompletionBridgeTransformationHandler: return self._convert_pcm16_to_wav(decoded_audio), "audio/wav" def transform_response( - self, model_response: "ModelResponse", response_format: str | None + self, model_response: "ModelResponse", model: str, response_format: str | None ) -> "HttpxBinaryResponseContent": import base64 @@ -176,10 +222,9 @@ class SpeechToCompletionBridgeTransformationHandler: raise ValueError("No audio part found in the response") decoded_audio: Final = base64.b64decode(audio_part.data) - model: Final = getattr(model_response, "model", "") content, content_type = ( self._gemini_tts_response_body(decoded_audio, response_format) - if self._is_gemini_tts_model(model) + if self._is_gemini_tts_model(model) or self._is_gemini_tts_model(model_response.model or "") else (decoded_audio, "audio/mpeg") ) response: Final = httpx.Response( diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index 176569f5d8d..d1e3f10fdd4 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -8,6 +8,7 @@ import time from collections.abc import AsyncGenerator, Callable, Iterable, Iterator, Mapping from contextlib import aclosing from dataclasses import dataclass +from functools import partial from types import MappingProxyType from typing import IO, Any, Final, TypedDict from urllib.parse import quote, unquote @@ -30,6 +31,7 @@ from litellm.litellm_core_utils.cloud_storage_security import ( split_configured_cloud_bucket_name, validate_managed_cloud_file_id, ) +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, @@ -1103,6 +1105,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): def _map_openai_to_vertex_params( self, openai_request_body: dict[str, Any], + deployment_drop_params: object = None, ) -> dict[str, object]: """ wrapper to call VertexGeminiConfig.map_openai_params @@ -1121,7 +1124,7 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): model=model, non_default_params=openai_request_body, optional_params={}, - drop_params=False, + drop_params=drops_unsupported_params(openai_request_body.get("drop_params", deployment_drop_params)), allow_integer_enums=False, ) return vertex_params @@ -1157,11 +1160,12 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): create_file_data=create_file_data, content_type=content_type, ): + deployment_drop_params: Final[object] = litellm_params.get("drop_params") return { "streaming_media_upload": StreamingMediaUploadConfig( body_stream=_OpenAIToVertexBatchUploadStream( file_data, - self._map_openai_to_vertex_params, + partial(self._map_openai_to_vertex_params, deployment_drop_params=deployment_drop_params), ), content_type="application/json", ) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 241b2d2a3a0..ce632bc0eef 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -12,7 +12,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args import httpx -from pydantic import JsonValue +from pydantic import JsonValue, TypeAdapter import litellm from litellm import verbose_logger @@ -28,7 +28,7 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO, ) from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -232,6 +232,274 @@ def _served_model_name(model_version: object) -> str | None: return model_version.split("@", 1)[0] +_GEMINI_SPEECH_CONFIG_KEY_MAP: Final[Mapping[str, str]] = MappingProxyType( + { + "multi_speaker_voice_config": "multiSpeakerVoiceConfig", + "speaker_voice_configs": "speakerVoiceConfigs", + "voice_config": "voiceConfig", + "prebuilt_voice_config": "prebuiltVoiceConfig", + "voice_name": "voiceName", + "replicated_voice_config": "replicatedVoiceConfig", + "mime_type": "mimeType", + "voice_sample_audio": "voiceSampleAudio", + "language_code": "languageCode", + "speaker_alias": "speakerAlias", + "speaker_id": "speakerId", + } +) + + +_SPEECH_CONFIG_ITEMS: Final = TypeAdapter(list[object]) + + +def _normalize_gemini_speech_config_item(value: Mapping[str, object]) -> Mapping[str, object]: + normalized: Final[dict[str, object]] = {} # mutable-ok: iterative traversal fills nested output without recursion + + pending: Final[ # mutable-ok: the explicit worklist must grow while avoiding recursive traversal + list[tuple[dict[str, object] | list[object], Mapping[str, object] | list[object]]] + ] = [(normalized, value)] + while pending: + target, source = pending.pop() + items = source.items() if isinstance(source, Mapping) else enumerate(source) + for key, item in items: + if isinstance(source, Mapping): + if not isinstance(key, str): + continue + normalized_key: str | int = _GEMINI_SPEECH_CONFIG_KEY_MAP.get(key, key) + else: + normalized_key = len(target) + + child: object + if (mapping_item := as_str_mapping(item)) is not None: + pending.append((child_mapping := dict[str, object](), mapping_item)) + child = child_mapping + elif isinstance(item, list): + pending.append((child_items := list[object](), _SPEECH_CONFIG_ITEMS.validate_python(item))) + child = child_items + else: + child = ( + OPENAI_TO_GEMINI_TTS_VOICES.get(item.lower(), item) + if normalized_key == "voiceName" and isinstance(item, str) + else item + ) + + if isinstance(target, dict): + assert isinstance(normalized_key, str) + target[normalized_key] = child + else: + target.append(child) + + return normalized + + +def _gemini_speaker_voice_config(item: object) -> object: + speaker_config: Final = as_str_mapping(item) + if speaker_config is None: + return item + speaker: Final = speaker_config.get("speakerAlias") + voice_name: Final = speaker_config.get("speakerId") + if not isinstance(speaker, str) or not isinstance(voice_name, str): + return item + return { + **{key: value for key, value in speaker_config.items() if key not in ("speakerAlias", "speakerId")}, + "speaker": speaker, + "voiceConfig": { + "prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES.get(voice_name.lower(), voice_name)} + }, + } + + +def _with_gemini_speaker_voice_configs(config: Mapping[str, object]) -> Mapping[str, object]: + multi_speaker_config: Final = as_str_mapping(config.get("multiSpeakerVoiceConfig")) + if multi_speaker_config is None: + return config + speaker_voice_configs: Final = multi_speaker_config.get("speakerVoiceConfigs") + if not isinstance(speaker_voice_configs, list): + return config + return { + **config, + "multiSpeakerVoiceConfig": { + **multi_speaker_config, + "speakerVoiceConfigs": [ + _gemini_speaker_voice_config(item) + for item in _SPEECH_CONFIG_ITEMS.validate_python(speaker_voice_configs) + ], + }, + } + + +_GEMINI_SPEECH_CONFIG_FIELDS: Final = frozenset({"voiceConfig", "multiSpeakerVoiceConfig", "languageCode"}) + +_GEMINI_VOICE_CONFIG_FIELDS: Final = MappingProxyType( + { + "prebuiltVoiceConfig": frozenset({"voiceName"}), + "replicatedVoiceConfig": frozenset({"mimeType", "voiceSampleAudio"}), + "voice": None, + } +) + +_GEMINI_SPEAKER_VOICE_CONFIG_FIELDS: Final = frozenset({"speaker", "voiceConfig"}) + + +def _known_leaf_fields(value: object, fields: frozenset[str] | None) -> object: + mapping: Final = as_str_mapping(value) + if fields is None or mapping is None: + return value + return {key: item for key, item in mapping.items() if key in fields} + + +def _known_voice_config_fields(value: object) -> object: + mapping: Final = as_str_mapping(value) + if mapping is None: + return value + return { + key: _known_leaf_fields(item, _GEMINI_VOICE_CONFIG_FIELDS[key]) + for key, item in mapping.items() + if key in _GEMINI_VOICE_CONFIG_FIELDS + } + + +def _known_speaker_voice_config_fields(value: object) -> object: + mapping: Final = as_str_mapping(value) + if mapping is None: + return value + return { + key: _known_voice_config_fields(item) if key == "voiceConfig" else item + for key, item in mapping.items() + if key in _GEMINI_SPEAKER_VOICE_CONFIG_FIELDS + } + + +def _known_speaker_voice_configs(value: object) -> object: + if not isinstance(value, list): + return value + return [_known_speaker_voice_config_fields(item) for item in _SPEECH_CONFIG_ITEMS.validate_python(value)] + + +def _known_multi_speaker_voice_config_fields(value: object) -> object: + mapping: Final = as_str_mapping(value) + if mapping is None: + return value + return {key: _known_speaker_voice_configs(item) for key, item in mapping.items() if key == "speakerVoiceConfigs"} + + +def _known_speech_config_field(key: str, value: object) -> object: + match key: + case "voiceConfig": + return _known_voice_config_fields(value) + case "multiSpeakerVoiceConfig": + return _known_multi_speaker_voice_config_fields(value) + case _: + return value + + +def _known_gemini_speech_fields( + config: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict + return { + key: _known_speech_config_field(key, item) + for key, item in config.items() + if key in _GEMINI_SPEECH_CONFIG_FIELDS + } + + +_GEMINI_TTS_VOICE_KEYS: Final = frozenset( + { + "format", + "name", + "voice", + "voiceName", + "voice_name", + "languageCode", + "language_code", + "modelName", + "model_name", + "speechConfig", + "speech_config", + "voiceConfig", + "voice_config", + "multiSpeakerVoiceConfig", + "multi_speaker_voice_config", + } +) + + +_NAMED_UNKNOWN_VOICE_KEYS: Final = 10 + +_NAMED_VOICE_KEY_CHARS: Final = 100 + + +def _gemini_speech_config_wrapper(value: Mapping[str, object]) -> Mapping[str, object] | None: + camel_config: Final = as_str_mapping(value.get("speechConfig")) + return camel_config if camel_config is not None else as_str_mapping(value.get("speech_config")) + + +def named_voice_keys(keys: Sequence[str]) -> str: + named: Final = ", ".join(key[:_NAMED_VOICE_KEY_CHARS] for key in keys[:_NAMED_UNKNOWN_VOICE_KEYS]) + unnamed_count: Final = len(keys) - _NAMED_UNKNOWN_VOICE_KEYS + return f"{named}, and {unnamed_count} more" if unnamed_count > 0 else named + + +def reject_unknown_gemini_tts_voice_keys( + model: str, voice: Mapping[str, object], drop_params: bool, llm_provider: str | None +) -> None: + nested_config: Final = _gemini_speech_config_wrapper(voice) or {} + ignored_wrapper: Final = ( + ("speech_config",) + if as_str_mapping(voice.get("speechConfig")) is not None and voice.get("speech_config") is not None + else () + ) + unknown_keys: Final = sorted( + {*(key for key in (*voice, *nested_config) if key not in _GEMINI_TTS_VOICE_KEYS), *ignored_wrapper} + ) + if not unknown_keys or drops_unsupported_params(drop_params): + return + raise UnsupportedParamsError( + message=( + f"{model} does not support the voice keys {named_voice_keys(unknown_keys)}. Remove them, or set " + "drop_params to ignore them." + ), + model=model, + llm_provider=llm_provider, + ) + + +def normalize_gemini_speech_config( + value: Mapping[str, object], drop_unknown_fields: bool = False +) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict + speech_config: Final = {**_with_gemini_speaker_voice_configs(mapped_gemini_speech_config(value))} + return _known_gemini_speech_fields(speech_config) if drop_unknown_fields else speech_config + + +def _merge_gemini_speech_config_wrapper(value: Mapping[str, object]) -> Mapping[str, object]: + nested_config: Final = _gemini_speech_config_wrapper(value) + if nested_config is None: + return value + return { + **{key: item for key, item in value.items() if key not in ("speechConfig", "speech_config")}, + **nested_config, + } + + +def mapped_gemini_speech_config( + value: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: provider request serialization requires a concrete dict + normalized: Final = _normalize_gemini_speech_config_item(_merge_gemini_speech_config_wrapper(value)) + speech_config: Final = {key: item for key, item in normalized.items() if key in _GEMINI_SPEECH_CONFIG_FIELDS} + voice_name: Final = next( + (normalized[key] for key in ("name", "voiceName", "voice") if isinstance(normalized.get(key), str)), + None, + ) + if "voiceConfig" in speech_config or "multiSpeakerVoiceConfig" in speech_config or not isinstance(voice_name, str): + return speech_config + return { + **speech_config, + "voiceConfig": { + "prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES.get(voice_name.lower(), voice_name)} + }, + } + + class VertexAIBaseConfig: def get_mapped_special_auth_params(self) -> dict: """ @@ -1237,7 +1505,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): status_code=400, ) - def _map_audio_params(self, value: dict) -> dict: + def _map_audio_params( + self, value: Mapping[str, object], drop_unknown_fields: bool = False + ) -> dict[str, object]: # mutable-ok: the public forwarding interface returns a concrete request dictionary """ Expected input: { @@ -1256,12 +1526,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): languageCode: "en-US", } """ - from litellm.types.llms.vertex_ai import ( - PrebuiltVoiceConfig, - SpeechConfig, - VoiceConfig, - ) - # Validate audio format - Gemini TTS only supports pcm16 audio_format: Final = value.get("format") if audio_format is not None and audio_format != "pcm16": @@ -1271,22 +1535,46 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): f"Please set audio format to 'pcm16'." ) - # Map OpenAI audio parameter to Gemini speech config - speech_config: Final[SpeechConfig] = {} + camel_speech_config: Final = value.get("speechConfig") + snake_speech_config: Final = value.get("speech_config") + speech_config_value: Final = ( + camel_speech_config if isinstance(camel_speech_config, Mapping) else snake_speech_config + ) + language_code: Final = value.get("language_code", value.get("languageCode")) + if isinstance(speech_config_value, Mapping): + return normalize_gemini_speech_config(value, drop_unknown_fields) - if "voice" in value: - voice: Final = value["voice"] - voice_name: Final = ( - OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice) if isinstance(voice, str) else voice - ) - prebuilt_voice_config: Final[PrebuiltVoiceConfig] = {"voiceName": voice_name} - voice_config: Final[VoiceConfig] = {"prebuiltVoiceConfig": prebuilt_voice_config} - speech_config["voiceConfig"] = voice_config + voice: Final = value.get("voice") + speech_config_without_language: Final[ # mutable-ok: provider request serialization requires a concrete dict + dict[str, object] + ] = ( + { + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice) + if isinstance(voice, str) + else voice, + } + } + } + if "voice" in value + else {} + ) - if "language_code" in value: - speech_config["languageCode"] = value["language_code"] + if language_code is not None: + return { + **speech_config_without_language, + "languageCode": language_code, + } - return cast(dict, speech_config) + return speech_config_without_language + + def _map_chat_audio_params( + self, model: str, value: Mapping[str, object], drop_params: bool + ) -> dict[str, object]: # mutable-ok: the public forwarding interface returns a concrete request dictionary + if _gemini_speech_config_wrapper(value) is not None: + reject_unknown_gemini_tts_voice_keys(model, value, drop_params, self.custom_llm_provider) + return self._map_audio_params(value, drop_unknown_fields=drops_unsupported_params(drop_params)) def map_audio_params( self, @@ -1351,7 +1639,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): elif param == "n": optional_params["candidate_count"] = value elif param == "audio" and isinstance(value, dict): - optional_params["speechConfig"] = self._map_audio_params(value) + optional_params["speechConfig"] = self._map_chat_audio_params(model, value, drop_params) elif param == "stop": if isinstance(value, str): optional_params["stop_sequences"] = [value] diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 2fae2a9de1c..1441d01e41e 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -6,20 +6,25 @@ Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/s """ import base64 -from collections.abc import Coroutine, Mapping +import math +from collections.abc import Coroutine, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Any, Final, TypeAlias, Union +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypeAlias, Union import httpx -from pydantic import ConfigDict, TypeAdapter +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError import litellm +from litellm._logging import verbose_logger from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import ( DEFAULT_SPEECH_MEDIA_TYPE, + calculate_request_duration, speech_media_type_from_audio_bytes, ) +from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, @@ -33,6 +38,7 @@ from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES from litellm.types.llms.vertex_ai_text_to_speech import ( VertexTextToSpeechAudioConfig, VertexTextToSpeechInput, + VertexTextToSpeechSpeakerVoiceConfig, VertexTextToSpeechVoice, ) @@ -49,6 +55,166 @@ _LyriaVoice: TypeAlias = str | dict | None _JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True)) _JSON_ARRAY: Final = TypeAdapter(tuple[object, ...], config=ConfigDict(hide_input_in_errors=True)) _STR: Final = TypeAdapter(str, config=ConfigDict(hide_input_in_errors=True)) +_GEMINI_CLOUD_TTS_LOCATIONS: Final = MappingProxyType( + { + **dict.fromkeys( + ("us", "us-central1", "us-east1", "us-east4", "us-east5", "us-south1", "us-west1", "us-west4"), "us" + ), + **dict.fromkeys( + ("eu", "europe-central2", "europe-north1", "europe-southwest1", "europe-west1", "europe-west4"), "eu" + ), + "northamerica-northeast1": "northamerica-northeast1", + } +) + + +_DEFAULT_GEMINI_TTS_VOICE: Final = "Kore" + + +class _GeminiCloudTtsInput(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str | None = None + ssml: str | None = None + prompt: str | None = None + + +class _GeminiCloudTtsAudioConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + audioEncoding: str = "" + sampleRateHertz: int | None = None + + +class _GeminiCloudTtsRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + input: _GeminiCloudTtsInput + audioConfig: _GeminiCloudTtsAudioConfig + + +def _fallback_gemini_tts_audio_duration(audio: bytes, encoding: str, sample_rate: int) -> float | None: + if encoding == "PCM": + return len(audio) / (2 * sample_rate) if sample_rate > 0 else None + if encoding in ("ALAW", "MULAW") and not (audio[:4] == b"RIFF" and audio[8:12] == b"WAVE"): + return len(audio) / sample_rate if sample_rate > 0 else None + + if encoding in ("LINEAR16", "ALAW", "MULAW") and audio[:4] == b"RIFF" and audio[8:12] == b"WAVE": + format_offset: Final = audio.find(b"fmt ", 12) + data_offset: Final = audio.find(b"data", 12) + if format_offset >= 0 and data_offset >= 0 and format_offset + 20 <= len(audio): + byte_rate: Final = int.from_bytes(audio[format_offset + 16 : format_offset + 20], "little") + data_size: Final = int.from_bytes(audio[data_offset + 4 : data_offset + 8], "little") + if byte_rate > 0 and data_size <= len(audio) - data_offset - 8: + return data_size / byte_rate + + if encoding == "MP3" and speech_media_type_from_audio_bytes(audio) == "audio/mpeg": + return len(audio) / 4000 + + if encoding == "OGG_OPUS" and audio[:4] == b"OggS": + opus_header: Final = audio.find(b"OpusHead") + last_page: Final = audio.rfind(b"OggS") + if opus_header >= 0 and opus_header + 12 <= len(audio) and last_page + 14 <= len(audio): + pre_skip: Final = int.from_bytes(audio[opus_header + 10 : opus_header + 12], "little") + granule: Final = int.from_bytes(audio[last_page + 6 : last_page + 14], "little") + if pre_skip <= granule < 2**64 - 1: + return (granule - pre_skip) / 48000 + + return None + + +_SPEAKER_CONFIG_ITEMS: Final = TypeAdapter(list[object]) + +_CloudTtsVoiceFields: TypeAlias = "Mapping[str, _CloudTtsVoiceFields]" + +_CLOUD_TTS_VOICE_LEAF: Final[_CloudTtsVoiceFields] = MappingProxyType({}) + +_CLOUD_TTS_PREBUILT_VOICE_FIELDS: Final[_CloudTtsVoiceFields] = MappingProxyType( + {"prebuiltVoiceConfig": MappingProxyType({"voiceName": _CLOUD_TTS_VOICE_LEAF})} +) + +_CLOUD_TTS_SPEECH_CONFIG_FIELDS: Final[_CloudTtsVoiceFields] = MappingProxyType( + { + "languageCode": _CLOUD_TTS_VOICE_LEAF, + "voiceConfig": _CLOUD_TTS_PREBUILT_VOICE_FIELDS, + "multiSpeakerVoiceConfig": MappingProxyType( + { + "speakerVoiceConfigs": MappingProxyType( + { + "speaker": _CLOUD_TTS_VOICE_LEAF, + "speakerAlias": _CLOUD_TTS_VOICE_LEAF, + "speakerId": _CLOUD_TTS_VOICE_LEAF, + "voiceConfig": _CLOUD_TTS_PREBUILT_VOICE_FIELDS, + } + ) + } + ), + } +) + + +_CLOUD_TTS_LIST_FIELDS: Final = frozenset({"speakerVoiceConfigs"}) + +_GEMINI_TTS_VOICE_NAME_KEYS: Final = ("name", "voiceName", "voice_name", "voice") + + +def _fields_cloud_tts_cannot_send( + config: Mapping[str, object], fields: _CloudTtsVoiceFields, path: str +) -> Iterator[str]: + for key, value in config.items(): + yield from _field_cloud_tts_cannot_send(f"{path}{key}", value, fields.get(key), key in _CLOUD_TTS_LIST_FIELDS) + + +def _field_cloud_tts_cannot_send( + path: str, value: object, fields: "_CloudTtsVoiceFields | None", is_list_field: bool +) -> Iterator[str]: + if fields is None: + yield path + return + if value is None: + return + if not fields: + yield from () if isinstance(value, str) else (path,) + return + items: Final = ( + _SPEAKER_CONFIG_ITEMS.validate_python(value) if is_list_field and isinstance(value, list) else (value,) + ) + for item in items: + mapping = as_str_mapping(item) + yield from (path,) if mapping is None else _fields_cloud_tts_cannot_send(mapping, fields, f"{path}.") + + +def _non_string_voice_names(config: Mapping[str, object], path: str) -> Iterator[str]: + return ( + f"{path}{key}" + for key in _GEMINI_TTS_VOICE_NAME_KEYS + if config.get(key) is not None and not isinstance(config.get(key), str) + ) + + +def _non_mapping_speech_config_wrappers(voice: Mapping[str, object]) -> Iterator[str]: + return ( + key + for key in ("speechConfig", "speech_config") + if voice.get(key) is not None and as_str_mapping(voice.get(key)) is None + ) + + +def _speaker_entries(config: Mapping[str, object]) -> Iterator[Mapping[str, object]]: + multi_speaker_config: Final = as_str_mapping(config.get("multiSpeakerVoiceConfig")) + speakers: Final = multi_speaker_config.get("speakerVoiceConfigs") if multi_speaker_config is not None else None + items: Final = _SPEAKER_CONFIG_ITEMS.validate_python(speakers) if isinstance(speakers, list) else () + return (entry for item in items if (entry := as_str_mapping(item)) is not None) + + +def _prebuilt_voices_replaced_by_speaker_id(config: Mapping[str, object]) -> Iterator[str]: + return ( + "multiSpeakerVoiceConfig.speakerVoiceConfigs.voiceConfig.prebuiltVoiceConfig" + for entry in _speaker_entries(config) + if entry.get("speakerId") is not None + and (voice_config := as_str_mapping(entry.get("voiceConfig"))) is not None + and voice_config.get("prebuiltVoiceConfig") is not None + ) class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): @@ -81,14 +247,19 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): } # Response format mappings from OpenAI to Google Cloud audio encoding - FORMAT_MAPPINGS = { - "mp3": "MP3", - "opus": "OGG_OPUS", - "aac": "MP3", # Google doesn't have AAC, use MP3 - "flac": "FLAC", - "wav": "LINEAR16", - "pcm": "LINEAR16", - } + FORMAT_MAPPINGS: ClassVar[Mapping[str, str]] = MappingProxyType( + { + "mp3": "MP3", + "opus": "OGG_OPUS", + "aac": "MP3", # Google doesn't have AAC, use MP3 + "flac": "FLAC", + "wav": "LINEAR16", + "pcm": "LINEAR16", + } + ) + GEMINI_FORMAT_MAPPINGS: ClassVar[Mapping[str, str]] = MappingProxyType( + {"mp3": "MP3", "opus": "OGG_OPUS", "alaw": "ALAW", "mulaw": "MULAW"} + ) def __init__(self) -> None: BaseTextToSpeechConfig.__init__(self) @@ -96,7 +267,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): def _map_voice_to_vertex_format( self, - voice: str | dict | None, + voice: str | dict[str, object] | None, + model: str | None = None, + drop_params: bool = False, ) -> tuple[str | None, dict | None]: """ Map voice to Vertex AI format. @@ -111,6 +284,9 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): - voice_str: Original string voice (for interface compatibility) - voice_dict: Vertex AI format dict with languageCode and name """ + if model is not None and self._is_gemini_tts_model(model): + return self._map_gemini_tts_voice_to_vertex_format(model=model, voice=voice, drop_params=drop_params) + if voice is None: return None, None @@ -142,11 +318,225 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): return voice_str, voice_dict + @staticmethod + def _is_gemini_tts_model(model: str) -> bool: + from litellm.utils import is_gemini_tts_model + + return is_gemini_tts_model(model) + + @staticmethod + def _get_str_value( + source: Mapping[str, object], + *keys: str, + ) -> str | None: + for key in keys: + value = source.get(key) + if isinstance(value, str): + return value + return None + + @staticmethod + def _get_dict_value( + source: Mapping[str, object], + *keys: str, + ) -> Mapping[str, object] | None: + for key in keys: + value = source.get(key) + if (mapping := as_str_mapping(value)) is not None: + return mapping + return None + + @staticmethod + def _resolve_gemini_tts_voice_name(model: str, voice_name: str, drop_params: bool) -> str: + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES + + gemini_voice_name: Final = OPENAI_TO_GEMINI_TTS_VOICES.get(voice_name.lower(), voice_name) + if "-" not in gemini_voice_name: + return gemini_voice_name + if drop_params: + return _DEFAULT_GEMINI_TTS_VOICE + raise UnsupportedParamsError( + message=( + f"{model} speaks Gemini voices such as {_DEFAULT_GEMINI_TTS_VOICE}, and {voice_name} is a Cloud " + "Text-to-Speech voice. Use a Gemini voice, or set drop_params to use the default Gemini voice." + ), + model=model, + llm_provider="vertex_ai", + ) + + @staticmethod + def _reject_malformed_gemini_tts_voice(model: str, drop_params: bool) -> None: + if drop_params: + return + raise UnsupportedParamsError( + message=( + f"{model} needs voice to be a Gemini voice name such as {_DEFAULT_GEMINI_TTS_VOICE} or a speech " + "config object. Send one, or set drop_params to use the default Gemini voice." + ), + model=model, + llm_provider="vertex_ai", + ) + + @staticmethod + def _reject_voice_fields_cloud_tts_cannot_send(model: str, voice: Mapping[str, object], drop_params: bool) -> None: + if drop_params: + return + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + mapped_gemini_speech_config, + named_voice_keys, + ) + + speech_config: Final = VertexAITextToSpeechConfig._get_dict_value(voice, "speechConfig", "speech_config") + mapped_config: Final = mapped_gemini_speech_config(voice) + unsendable: Final = sorted( + { + *_fields_cloud_tts_cannot_send(mapped_config, _CLOUD_TTS_SPEECH_CONFIG_FIELDS, ""), + *_prebuilt_voices_replaced_by_speaker_id(mapped_config), + *_non_mapping_speech_config_wrappers(voice), + *_non_string_voice_names(voice, ""), + *_non_string_voice_names(speech_config or {}, "speechConfig."), + } + ) + if not unsendable: + return + raise UnsupportedParamsError( + message=( + f"{model} cannot send the voice fields {named_voice_keys(unsendable)} as mp3, opus, alaw, or mulaw. " + "Request wav or pcm, remove them, or set drop_params to ignore them." + ), + model=model, + llm_provider="vertex_ai", + ) + + @staticmethod + def _reject_malformed_gemini_tts_speaker(model: str, drop_params: bool) -> None: + if drop_params: + return + raise UnsupportedParamsError( + message=( + f"{model} needs each entry of speaker_voice_configs to name a speaker and a voice, as speaker with " + "voice_config.prebuilt_voice_config.voice_name, or speakerAlias with speakerId." + ), + model=model, + llm_provider="vertex_ai", + ) + + def _extract_gemini_tts_speaker_configs( + self, + model: str, + speech_config: Mapping[str, object], + drop_params: bool, + ) -> Sequence[VertexTextToSpeechSpeakerVoiceConfig]: + multi_speaker_config: Final = self._get_dict_value(speech_config, "multiSpeakerVoiceConfig") + if multi_speaker_config is None: + return [] + raw_speaker_configs: Final = multi_speaker_config.get("speakerVoiceConfigs", []) + try: + validated_speaker_configs: Final = _SPEAKER_CONFIG_ITEMS.validate_python(raw_speaker_configs, strict=True) + except ValidationError: + self._reject_malformed_gemini_tts_speaker(model, drop_params) + return [] + speaker_configs: Final[ # mutable-ok: validated payloads are accumulated for serialization + list[VertexTextToSpeechSpeakerVoiceConfig] + ] = [] + for raw_item in validated_speaker_configs: + if (raw_config := as_str_mapping(raw_item)) is None: + self._reject_malformed_gemini_tts_speaker(model, drop_params) + continue + speaker_alias = self._get_str_value(raw_config, "speakerAlias", "speaker") + speaker_id = self._get_str_value(raw_config, "speakerId") or self._prebuilt_voice_name(raw_config) + if speaker_alias is None or speaker_id is None: + self._reject_malformed_gemini_tts_speaker(model, drop_params) + continue + speaker_configs.append( + { + "speakerAlias": speaker_alias, + "speakerId": self._resolve_gemini_tts_voice_name(model, speaker_id, drop_params), + } + ) + return speaker_configs + + def _prebuilt_voice_name(self, config: Mapping[str, object]) -> str | None: + voice_config: Final = self._get_dict_value(config, "voiceConfig") + prebuilt_voice_config: Final = ( + self._get_dict_value(voice_config, "prebuiltVoiceConfig") if voice_config is not None else None + ) + if prebuilt_voice_config is None: + return None + return self._get_str_value(prebuilt_voice_config, "voiceName") + + def _map_gemini_tts_voice_to_vertex_format( + self, + model: str, + voice: object, + drop_params: bool = False, + ) -> tuple[str | None, dict[str, object]]: # mutable-ok: provider request serialization requires a concrete dict + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + mapped_gemini_speech_config, + reject_unknown_gemini_tts_voice_keys, + ) + + default_voice: Final[dict[str, object]] = { + "languageCode": self.DEFAULT_LANGUAGE_CODE, + "modelName": model, + "name": _DEFAULT_GEMINI_TTS_VOICE, + } + if voice is None: + return None, default_voice + + if isinstance(voice, str) and voice: + return voice, { + "languageCode": self.DEFAULT_LANGUAGE_CODE, + "modelName": model, + "name": self._resolve_gemini_tts_voice_name(model, voice, drop_params), + } + + voice_config: Final = as_str_mapping(voice) + if voice_config is None: + self._reject_malformed_gemini_tts_voice(model, drop_params) + return None, default_voice + reject_unknown_gemini_tts_voice_keys(model, voice_config, drop_params, "vertex_ai") + + speech_config: Final = mapped_gemini_speech_config(voice_config) + language_code: Final = self._get_str_value(speech_config, "languageCode") or self.DEFAULT_LANGUAGE_CODE + model_name: Final = model + speaker_configs: Final = self._extract_gemini_tts_speaker_configs(model, speech_config, drop_params) + self._reject_voice_fields_cloud_tts_cannot_send(model, voice_config, drop_params) + if speaker_configs: + return None, { + "languageCode": language_code, + "modelName": model_name, + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": speaker_configs, + }, + } + voice_name: Final = self._prebuilt_voice_name(speech_config) + if voice_name is not None: + return None, { + "languageCode": language_code, + "modelName": model_name, + "name": self._resolve_gemini_tts_voice_name(model, voice_name, drop_params), + } + return None, { + "languageCode": language_code, + "modelName": model_name, + "name": _DEFAULT_GEMINI_TTS_VOICE, + } + + @staticmethod + def _dispatch_voice_name(voice: str | Mapping[str, object] | None) -> str | None: + if isinstance(voice, str): + return voice + if not isinstance(voice, Mapping): + return None + name: Final = voice.get("name") + return name if isinstance(name, str) else None + def dispatch_text_to_speech( self, model: str, input: str, - voice: str | dict | None, + voice: str | dict[str, object] | None, optional_params: dict[str, object], litellm_params_dict: dict[str, object], logging_obj: "LiteLLMLoggingObj", @@ -175,14 +565,17 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): vertex_project: Final = self.safe_get_vertex_ai_project(litellm_params_dict) vertex_location: Final = self.safe_get_vertex_ai_location(litellm_params_dict) - # Convert voice to string if it's a dict (extract name) - # Actual voice mapping happens in map_openai_params - voice_str: str | None = None - if isinstance(voice, str): - voice_str = voice - elif isinstance(voice, dict): - # Extract voice name from dict if needed - voice_str = voice.get("name") if voice else None + mapped_voice, mapped_params = ( + self.map_openai_params( + model=model, + voice=voice, + optional_params=optional_params, + drop_params=drops_unsupported_params(kwargs.get("drop_params")), + kwargs=kwargs, + ) + if self._is_gemini_tts_model(model) + else (self._dispatch_voice_name(voice), optional_params) + ) # Store credentials in litellm_params for use in transform methods litellm_params_dict.update( @@ -198,9 +591,11 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): response: Final = base_llm_http_handler.text_to_speech_handler( model=model, input=input, - voice=voice_str, - text_to_speech_provider_config=self, - text_to_speech_optional_params=optional_params, + voice=mapped_voice, + text_to_speech_provider_config=( + VertexAIGeminiCloudTextToSpeechConfig() if self._is_gemini_tts_model(model) else self + ), + text_to_speech_optional_params=mapped_params, custom_llm_provider="vertex_ai", litellm_params=litellm_params_dict, logging_obj=logging_obj, @@ -219,13 +614,13 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): Note: Vertex AI also supports additional parameters like audioConfig which can be passed but are not part of the OpenAI spec """ - return ["voice", "response_format", "speed"] + return ["voice", "response_format", "speed", *(["instructions"] if self._is_gemini_tts_model(model) else [])] def map_openai_params( self, model: str, optional_params: dict, - voice: str | dict | None = None, + voice: str | dict[str, object] | None = None, drop_params: bool = False, kwargs: dict = {}, ) -> tuple[str | None, dict]: @@ -244,19 +639,28 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): Tuple of (mapped_voice_str, mapped_params) """ mapped_params: Final[dict[str, object]] = {} + openai_params: Final = _JSON_OBJECT.validate_python(optional_params) + drops_params: Final = drop_params or litellm.drop_params is True ########################################################## # Map voice using helper ########################################################## - mapped_voice_str, voice_dict = self._map_voice_to_vertex_format(voice) + mapped_voice_str, voice_dict = self._map_voice_to_vertex_format( + voice=voice, + model=model, + drop_params=drops_params, + ) if voice_dict is not None: mapped_params["vertex_voice_dict"] = voice_dict # Map response format if "response_format" in optional_params: format_name: Final = optional_params["response_format"] - if format_name in self.FORMAT_MAPPINGS: - mapped_params["audioEncoding"] = self.FORMAT_MAPPINGS[format_name] + format_mappings: Final = ( + self.GEMINI_FORMAT_MAPPINGS if self._is_gemini_tts_model(model) else self.FORMAT_MAPPINGS + ) + if format_name in format_mappings: + mapped_params["audioEncoding"] = format_mappings[format_name] else: # Try to use it directly as Google Cloud format mapped_params["audioEncoding"] = format_name @@ -265,10 +669,19 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): mapped_params["audioEncoding"] = self.DEFAULT_AUDIO_ENCODING # Map speed (OpenAI: 0.25-4.0, Vertex AI: speakingRate 0.25-4.0) - if "speed" in optional_params: - speed: Final = optional_params["speed"] - if speed is not None: - mapped_params["speakingRate"] = str(speed) + speed: Final = openai_params.get("speed") + if speed is not None: + mapped_params["speakingRate"] = str(speed) + + instructions: Final = openai_params.get("instructions") + if isinstance(instructions, str) and self._is_gemini_tts_model(model): + mapped_params["prompt"] = instructions + elif instructions is not None and self._is_gemini_tts_model(model) and not drops_params: + raise UnsupportedParamsError( + message=f"{model} needs instructions to be a string. Send one, or set drop_params to ignore it.", + model=model, + llm_provider="vertex_ai", + ) # Pass through Vertex AI-specific parameters from kwargs if "audioConfig" in kwargs: @@ -311,15 +724,48 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): Google Cloud TTS endpoint: https://texttospeech.googleapis.com/v1/text:synthesize """ + if self._is_gemini_tts_model(model): + return self._get_gemini_tts_url( + model=model, api_base=api_base, litellm_params=_JSON_OBJECT.validate_python(litellm_params) + ) + if api_base: return api_base return self.TTS_API_URL + def _get_gemini_tts_url(self, model: str, api_base: str | None, litellm_params: Mapping[str, object]) -> str: + if api_base: + raise UnsupportedParamsError( + message=( + f"{model} serves mp3, opus, alaw and mulaw through Cloud Text-to-Speech, which does not use " + "this deployment's api_base. Request wav or pcm, or use a deployment without api_base." + ), + model=model, + llm_provider="vertex_ai", + ) + location: Final = litellm_params.get("vertex_location") + vertex_location: Final = location if isinstance(location, str) else self.get_default_vertex_location() + if vertex_location == "global": + return self.TTS_API_URL + cloud_tts_location: Final = _GEMINI_CLOUD_TTS_LOCATIONS.get(vertex_location) + if cloud_tts_location is None: + raise UnsupportedParamsError( + message=( + f"{model} serves mp3, opus, alaw and mulaw through Cloud Text-to-Speech, which has no Gemini TTS " + f"location for vertex_location {vertex_location}. Use a deployment in global, us, eu, " + "northamerica-northeast1, or a US or EU Vertex AI region." + ), + model=model, + llm_provider="vertex_ai", + ) + return f"https://{cloud_tts_location}-texttospeech.googleapis.com/v1/text:synthesize" + def _validate_vertex_input( self, input_data: VertexTextToSpeechInput, optional_params: dict, + detect_ssml: bool, ) -> VertexTextToSpeechInput: """ Validate and transform input for Vertex AI TTS @@ -343,7 +789,7 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): else: # LiteLLM will auto-detect if text is in ssml format # check if "text" is an ssml - in this case we should pass it as ssml instead of text - if input_data: + if detect_ssml and input_data: _text: Final = input_data.get("text", None) or "" if "" in _text: input_data["ssml"] = input_data.pop("text") @@ -403,8 +849,13 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): headers["x-goog-user-project"] = vertex_project ####### Build the request ################ - vertex_input = VertexTextToSpeechInput(text=input) - vertex_input = self._validate_vertex_input(vertex_input, optional_params) + prompt: Final = _JSON_OBJECT.validate_python(optional_params).get("prompt") + validated_input: Final = self._validate_vertex_input( + VertexTextToSpeechInput(text=input), optional_params, detect_ssml=not self._is_gemini_tts_model(model) + ) + vertex_input: Final[VertexTextToSpeechInput] = ( + {**validated_input, "prompt": prompt} if isinstance(prompt, str) else validated_input + ) # Build voice configuration # Check for voice dict stored in: @@ -412,8 +863,8 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): # 2. optional_params by map_openai_params voice_dict: Final = litellm_params.get("vertex_voice_dict") or optional_params.get("vertex_voice_dict") if voice_dict is not None and isinstance(voice_dict, dict): - vertex_voice = VertexTextToSpeechVoice(**voice_dict) - elif voice is not None and isinstance(voice, str): + vertex_voice = _JSON_OBJECT.validate_python(voice_dict) + elif voice is not None: # Handle string voice (shouldn't normally happen if dispatch was called) parts: Final = voice.split("-") if len(parts) >= 2: @@ -485,8 +936,44 @@ class VertexAITextToSpeechConfig(BaseTextToSpeechConfig, VertexBase): content=binary_data, ) - # Initialize the HttpxBinaryResponseContent instance - return HttpxBinaryResponseContent(response) + binary_response: Final = HttpxBinaryResponseContent(response) + if self._is_gemini_tts_model(model): + from litellm.types.utils import CompletionTokensDetailsWrapper, Usage + + request: Final = _GeminiCloudTtsRequest.model_validate_json(raw_response.request.content) + input_data: Final = request.input + audio_config: Final = request.audioConfig + container_duration: Final = calculate_request_duration(binary_data) + sample_rate: Final = audio_config.sampleRateHertz or 24000 + duration: Final = ( + container_duration + if container_duration is not None + else _fallback_gemini_tts_audio_duration(binary_data, audio_config.audioEncoding, sample_rate) + ) + if duration is None: + verbose_logger.warning( + "Could not measure Gemini TTS audio duration for %s, so its audio tokens are not billed", model + ) + return binary_response + input_text: Final = " ".join( + value for value in (input_data.text, input_data.ssml, input_data.prompt) if value + ) + prompt_tokens: Final = litellm.token_counter(model=model, text=input_text) + audio_tokens: Final = math.ceil(duration * 25) + binary_response.usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=audio_tokens, + total_tokens=prompt_tokens + audio_tokens, + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=audio_tokens), + ) + return binary_response + + +class VertexAIGeminiCloudTextToSpeechConfig(VertexAITextToSpeechConfig): + def get_error_class(self, error_message: str, status_code: int, headers: dict[str, str]) -> BaseLLMException: + return super().get_error_class( + error_message=error_message, status_code=400 if status_code == 404 else status_code, headers=headers + ) class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): diff --git a/litellm/main.py b/litellm/main.py index d647cab24e3..079b0bcbd6c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8846,8 +8846,11 @@ def speech( generic_optional_params: Final = GenericLiteLLMParams.model_validate(kwargs) - # Handle Gemini models separately (they use speech_to_completion_bridge) - if "gemini" in model: + needs_cloud_text_to_speech: Final = ( + litellm.utils.is_gemini_tts_model(model) + and response_format in VertexAITextToSpeechConfig.GEMINI_FORMAT_MAPPINGS + ) + if "gemini" in model and not needs_cloud_text_to_speech: from .endpoints.speech.speech_to_completion_bridge.handler import ( speech_to_completion_bridge_handler, ) diff --git a/litellm/proxy/openai_files_endpoints/common_utils.py b/litellm/proxy/openai_files_endpoints/common_utils.py index 2e07694876b..f7289445d3a 100644 --- a/litellm/proxy/openai_files_endpoints/common_utils.py +++ b/litellm/proxy/openai_files_endpoints/common_utils.py @@ -477,6 +477,18 @@ async def get_authorized_credentials_for_model( ) +def deployment_drop_params( + llm_router: Optional["Router"], model_id: str, request: Mapping[str, object] +) -> Mapping[str, bool | str]: + deployment: Final = ( + llm_router.get_credential_deployment(model_id=model_id) + if llm_router is not None and "drop_params" not in request + else None + ) + drop_params: Final = deployment.litellm_params.drop_params if deployment is not None else None + return MappingProxyType({}) if drop_params is None else MappingProxyType({"drop_params": drop_params}) + + def _key_allowlist_covers(public_model_name: str, key_model_allowlist: Sequence[str]) -> bool: from litellm.proxy.auth.auth_checks import model_matches_any_wildcard_pattern_in_list diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 8823f01d781..d2ceefffe47 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -76,6 +76,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( # noqa: F401 # add_internal_model_credentials, apply_team_provider_credentials, authorize_model_for_key, + deployment_drop_params, encode_file_id_with_model, extract_file_creation_params, get_authorized_credentials_for_model, @@ -457,7 +458,7 @@ async def route_create_file( # Merge credentials into the request prepare_data_with_credentials( data=_create_file_request, - credentials=credentials, + credentials={**credentials, **deployment_drop_params(llm_router, model, _create_file_request)}, ) # Create the file with model credentials diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index 0480d6b1f3e..f1176f777c7 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -239,8 +239,18 @@ class VoiceConfig(TypedDict): prebuiltVoiceConfig: PrebuiltVoiceConfig +class SpeakerVoiceConfig(TypedDict): + speaker: ReadOnly[str] + voiceConfig: ReadOnly[VoiceConfig] + + +class MultiSpeakerVoiceConfig(TypedDict): + speakerVoiceConfigs: ReadOnly[Sequence[SpeakerVoiceConfig]] + + class SpeechConfig(TypedDict, total=False): voiceConfig: VoiceConfig + multiSpeakerVoiceConfig: ReadOnly[MultiSpeakerVoiceConfig] languageCode: str diff --git a/litellm/types/llms/vertex_ai_text_to_speech.py b/litellm/types/llms/vertex_ai_text_to_speech.py index e8f9fbe4a24..86b131dce59 100644 --- a/litellm/types/llms/vertex_ai_text_to_speech.py +++ b/litellm/types/llms/vertex_ai_text_to_speech.py @@ -4,7 +4,9 @@ Type definitions for Vertex AI Text-to-Speech API Reference: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/synthesize """ -from typing_extensions import TypedDict +from collections.abc import Sequence + +from typing_extensions import ReadOnly, TypedDict class VertexTextToSpeechInput(TypedDict, total=False): @@ -16,6 +18,16 @@ class VertexTextToSpeechInput(TypedDict, total=False): text: str | None ssml: str | None + prompt: ReadOnly[str | None] + + +class VertexTextToSpeechSpeakerVoiceConfig(TypedDict): + speakerAlias: ReadOnly[str] + speakerId: ReadOnly[str] + + +class VertexTextToSpeechMultiSpeakerVoiceConfig(TypedDict): + speakerVoiceConfigs: ReadOnly[Sequence[VertexTextToSpeechSpeakerVoiceConfig]] class VertexTextToSpeechVoice(TypedDict, total=False): @@ -29,6 +41,8 @@ class VertexTextToSpeechVoice(TypedDict, total=False): languageCode: str name: str + modelName: ReadOnly[str] + multiSpeakerVoiceConfig: ReadOnly[VertexTextToSpeechMultiSpeakerVoiceConfig] class VertexTextToSpeechAudioConfig(TypedDict, total=False): @@ -42,6 +56,7 @@ class VertexTextToSpeechAudioConfig(TypedDict, total=False): audioEncoding: str speakingRate: str + sampleRateHertz: ReadOnly[int] class VertexTextToSpeechRequest(TypedDict, total=False): diff --git a/litellm/utils.py b/litellm/utils.py index 1a114ffe9f0..7de7ce6188d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -860,6 +860,11 @@ def _is_gemini_model(model: str | None, custom_llm_provider: str | None) -> bool return model is not None and "gemini" in model.lower() +def is_gemini_tts_model(model: str) -> bool: + model_name: Final = model.lower() + return "gemini" in model_name and "tts" in model_name + + def _remove_thought_signature_from_id(tool_call_id: str, separator: str) -> str: """ Remove thought signature from a tool call ID. diff --git a/tests/audio_tests/gemini_tts_speech.mp3 b/tests/audio_tests/gemini_tts_speech.mp3 new file mode 100644 index 0000000000000000000000000000000000000000..71a916ad6c0803a578e56162e8d97996d999b228 GIT binary patch literal 4992 zcmeI#cTiK?x&ZLK6TlE!0s$fdhN6b1p$P~G0U}K$Vi2&=5wL|S7ETbUL8T+oq>B+1 z1Tlh05fB0CHUt4hsRtB6kRw?y@ywhvZ|?i+zI*<=-^`xbv-a%Y`sQ2Tti2cbX3-2` zrozHua}2zV0b;Xi8?;SfoPMQ2Ph|GD@s|Mn7)DulJAx&{{9yzZ|7CepQAy#aHW+_e zIEq)g2LM=+6N^qnYCru6bsC-fX6AYpIbyvbcfn)bYkx1m^~EIp><`27L{yRG9phay1Y`NEPBEGIaQ2#u1rsCn%xv5{`QpN46tG03i! z`c2h$W-N?$(|}q3xRjettV&5bE$igU@_}7}vBbEMf*7jd%<-!!}WthPrLz?uWIbs2F<<>GvG* z-PcHujIWM>19PMhjn~KR&fPd~qjO!o@Mueu?sxD500yZb>W~)=*$l6eM_Cz0L|1o+ z=ZoTc5IXQBJd9kHg|xrfSm%+T2G}IgN?i+1d=}1Fp#{LHiHXKr8uiMV%_Wi)H*W@Eiwv&@`CVp)N*ii}G2!T!* z*4&lAVBWG!^+pN%^Q+Kd*uJyzhPRgB_DI~X{3UOrE#L?uoR-qqLp@te8Nm~`&YNd? ztWk!1vYIK>mhW$TvMOvWf+bbbQq~5>TI~&!dTQ=J@*Y`Byh^Y}b+&&LN|V;SH4Yd% zX1A^ys7ggXZ-V_c%V1S_J{o7wBEd2GP@Hme&tVAry^PFdBOD7aqjf?Uq zaFaa)zLXCI2@1L=k;?W15EUP(WL8)#D4OBO?NLYF!+wR{;&Fn?z;F7^m_^-aL~h_^ zAwQlkgl!wNf+H{x+deuaXUqowF!xN34ZaNhG5##6r?jJc#D}f5r^|qmAC9m!@$Zj5vQi;F~_NR%N0Ey51uX@k|4okw2A|^242wDGOyH_{mLKL$g+Z?@leQ(Tf$Cy_^<=5 zC9e5c+?wvso^YJlpq}G*1%fx{o6FJDjte@n1cFeSlOTzN5P! z6iU&P&>UcV{d#in!L&MYBU95R5U6&)ddA>p>U@BhLP_&yhC{A-?NsEsixHy^CrEAn zu2PCQaHQ^p+IGFq$&Q!@t$LE_Pg00Yje0abp|?kPz9yz2`2u_x5BVuCP$Co^IQPyh za!Ifvu*sR++m86O!vgnWV2?Q--rq7-@v2sd9~xGfiFLh7G(ria%#|8Zjgva!U%s`J zF!Agir=#p*cE%ja=qB*BRt@revTT!L6@1+Zx{|ftZe%fD%%FeUF}==rs~<8dCJf4J zRKIN$KJGoj02B6wFOrQk00Ui)a4OHx{XL=Mq5VMr+0`fIgiYqo2P@JoFrr4uqI_@v zSZq5uS9^B`=J^U(!R=M>TOx!{DjGPQRe5F#d>?XtlaE!#2d_X9{)~z$^+N!x7G20} zRqa6QyK*#2?a0toq(;!)l3v}fN?Jyu%|AW{$ZtHTrnEu5FvsNG?K|B4ZuV{aL7u-2 ztHLaUIwMoBznxE^NOciBOiTzz%qQvrm_p@JhJBBx(kYMoYWB-$a_+$>Xr0VWlGsXa zY+SxPa@IcDGxYxRL2vJ_$lj$OW9~aZyD%+D!wg-)7-CKGd}DFn)-q@yGSvek;{slT zwY|RQCF~!(3H45C-F!yW`}jts+om3sOWMD;Zppc}7eOcof5Z1yI^F#uf)tTD?j!)Yvro4J&+~U--V_u;U4-2isrwcJ`soJ;XBz_8^V=R~ zHJ-m4&F=di$2@K_bo$2kSyRQ?Oz$nV>1}BK#?J~paEEmS-XnVF{$a(04(^9E_~6y> z*P_SO&l+1lMCulT%f% zS|H1)oc#~#>dFJ}_itl8tW#L+77!bFEt9?T+mth<8|qEB7pVszr`QDmIfjsz;B&k@ zpTZx>Jl{(^-=+=nu}?i@_)^E(W5CBhe4lu9@RLIo*s?&C5u*?5BQ?Id6%ntZa!-s^ z5yl#hFgY=FXR*$vPA@OZ)xvXOa|-gcpd`dWA$7Nayia0zXkcomZC+n?U@Xr+iG{U# zAbl#G-m|wuhQ%)5*W(RH69AkXKJw)$_v zPMtsM+%&7@M6szCjD{D;HAwO z(jxL;i2=Z}-@-^i$A&alMQT@jcj`WQCHyI~am)wNsBS3-KYC7Ux&yKLW>?$q{_ZiB zat;AmX>xLEwXZu_ZOZD3SORO%1aO1vGsXQ?tDycqIsl$tj0IgVY&$Q`(KXgw5foX- z?ke7(V8@^A=!>aCph_hA>~vFFlUdGB&eT8t$OAYnk81u7vp+0KS(` zPl6sQX{w+*sL@Lx(Ze_XsRq$8P zakwLU2WZ9dVaNzaw*wI ztoU1e^U&cdz35RjmSSjb;(KpR#Dxv{Jii9}s34J-|9&+4l$ch0$4rRl9$K;wZ9aIu zc37Ibuice+q7ik)7|+`1g-SB}h)itrOW1KY)amSlP@m|#v7Jd9WH(_V`B+)9JJz4K z0V`4Wa%(?dOS-oAD9>+|ZfY%pY?|3=vxh74Pd(ppd9X8?kJu~i z9^f65pxVX!erC1)cL3G`y}p?nP-%FAJ8|#o=%#RMR1Dj7&&k`1OLk>JSaf6ymDTQE zxbiE1Sv7Piq(*(@dQqTSygUL0lt7F{HUkC2wz-wz1>PouJP*h{DMOLEi3AM-_or2hT6_NKO3u%$%ce1 zQp18W4_8VlES!?kZc*8J8jU;cvYfEgs|UcMW{GBTzlN8FB$JAFAJB^m12uBt53xIvA$y2@WCDic)!QkC^Ph&-=u4Usi~ zo5f_RlMDm=ELBBca#9sSn!emk-b2yes!1X_I5_-y&+|VjZcx>SFO#4@A8?=jCxX#- zB3#s*=ik9jYLVb32=at>X$8h+Jx1z4d)~t}5Guc=L63LXV+*JgAg1hLBt0BK#LFUs zAgpP_8{r;^BC7s06WS!IKGZADt_+NI7@C*3I{}use*;lRK&itB&o6@@q!q*s3+{kL z9^dHc0xBO}lTLuGS~oTb>{Ul(o0f|Wg}cHg7la`U{YKaqa+837#PUeA018EP-3Gpt zAYc*(HR|x%1;oixliZ`F^cy^09|``oKmX-dzb6Pgg=>eyF%VJ!V;oFr-x~Lfc4j+|FJXBQnPGzW?c2*f5hYQ3V+ zSlp8VH-8{gyWR>W<6-b)S=Jpn{+x*t;&)%#AD(~Y>F9Q=4DV-joW4j)TT?iR9i%k< zVX`TIOu;pw?e2MEkjJ`cj$NlBVV_bKShJ!Tq(*T|qmmkBe%(q5{860F^CxgJ)5^T_ z#bmQ%Z}U}si-;!kW($f%8pyw*Vrv$jcA3xD97dD!6hqJm(F%aXp=gjyLtW;PDv#aj?#xjB>8O-H_3UW>VGvF z>)(#Q`*TK8d|C&NBJ%q4-{$`1#lQV23H>D;+=Tz9{J;9=AIkr~{=Y0Pci`{!{b%a) T=bZm*|M|bt_t2m^|QgoM(m zln4j{FP`&wp7Y_k-gVu3U-#a>|9{Xx3iVgh`EVtj%E0>359 z*O$#>q>Hu96X|P+yoSPp*HDPf$==H4+WB|pptft>b^Ol!yDC+*21W=6$4pH@T0lVP z-d!m^5fNTJOJk(5HUeR2W}%~}g+Rdd;6{)1byd}MG!^AUc{$nGX09%;7eKP9oSI5z zmXRP2JJCH*ygkViZtp8Mt0Sgo{uPxqd8WtBUBj}?(pdGvdsFh|6+guQbnsgV{OYeB z{8h*Qa|42qWRLK2ipLcrC?2c z$K*k~wy`ZmfWL8;#8kUxkak}^&^kiHhBiP$4hV90AN(NYprEptMQDmeF8Yc`aq9M* z&*jTbqCxiyWBI1}8I1I4>0HzHVLhDe?9+T2(3s*-jE3`pc(^-{jjdi%Et3QT(}R*AF{hxudg40IT)G zd-PVx`Z0IMcB!#7lV zzhGO5gOtML;MN1plP$$AG{6jVAn-h3D(Y9b(TR(B;JEU&=567!5D36}@rozy{PlE# zz_EsU$;k;~(8i0$_{!UTK2S^hW=i0)$~>(SsS-P?N<`oJC_Z{8*{%ie_F6KguM0^{Mw+VCZ2ojr4*etO4Amr;EgGd>Wu`)jfKtfEY z%|@D+weJpZW%FVcwycF>?&u5Z)&%x5>2c>Jjdvl5AAZik>wmQbf&ie$wEah!X+|!g z?4K6$nc!UlS*h(LBlFqVM_xXG)nGH*Wtbe9ER7WN;woR`cm~xZ830;ge`mrVd16hi zId*H{#SLWS%hbZ$#tWGxMZx`uJJD)+kHa?@RI4Lk(Cmpy9LhTI3_!3D!+hxYn0+kW z5&VE7(`T8GeeEV~3Mxij*3M!AQD}^S9J^IM$->aF!cuR*CSv7@w9L6fHuAu*o~tun zD1_@kmMfVMTOba|MO@Z36xAY=N}s^dMQ#kWd~B!^S;N7a za~*H`v7cV1(J0!yhY8)*jj1`El)--0*8VFD|ILU9*w(}-NY@G|SZs3Odje*UT69ig zdRBJ|NG6ej;QzopKxT59Gtfw^;?>(mK=lHW1t`ayN2GC2h*0-wfS~+69mL zs+$MA>2CA8t?cZ@Dm{nl?+NeVZASlOxg5x*w7p{}O{5plexUdr$Qsb^4SQ(tb&V`V zV*Z59wj)}j+#~pm>xUqR&GZlCbSX?6H^lmxJm+lDj`+^zw8@OU##NfqR|Ko0q`^kh zF6Xa(maSP_sw7!XYTungcKu7^*puKOR7Us=e<_J!lDj(4n7!oDH)n{QGDo8wtx~S; z1IMb~D z^x*4+pi#LHS9mtKs4iFT&67N!`kmwf{_dd4(Kq2L{xPUDJH&5s+PwMfMKBHYGX;L+ z`5kknS(>g+8~8!ET}kVR^{|$c8`;!r9M(>Z0XWX8cV9!BIr}yp0Sx@)nf?|~uI&8- z=9?e?HN1Y85H0KA)GU6}(PtQoZ)ulxf>WzwX4iyGp1XC!k!nx%|CA4|Ue+(2L~{ganH$ ziR=*<7xrICiVq2Y#sxfkUR{__PKY3+Nh82voC#@swo}Cop&&$%v>b3I?8^(H4oD#Ge=X{3qzo%68 zM89QKyVy37BUX69~XDGfKFlC8{aYr?s5=xgr1;s4Hi-~BOC>_sg>!+&q6CCK@F?; zf^N#`dwc!eFRa%WiuuUoS?0z)qEha%YWYcEOpgS%OSR}?ECetL(yetsgUzG4M|X6B zpZ2I2QXQ1<^#vok;TXpQ(|e?`w8kFR$gPFa@1O+$D5#nzG{nSs_>#pBIFhL0JzB&I zx2fQF&!C2guelEUjP4$~(L-UuaQNb4tLLOeXM=-ira6$h60gV4_yD|g7@nX!xWusG z8Ld3A>bKW}tq!3hw^`syhvD?{;cJ#)ch7eORvzM~MN_^JhZ)acUMSE`c#cHcdWqmc zQYL4>k*&JeD^5U~=!$=};*+raC^}H3$CUBT_PHsEXXVE?OwPE7Vl+}LHLdK?VECm= z4?-Fgn0gBF>T!uGI(6GktkZUCH2;=>u9DhRBeFZ^tP!|89c6sA1Egh1&;JsjpYQkn z9N{HIlrZ#?bK`NT>Pz!Q^qh??g$H169a4h3yF{`m&{<~mc}z`^krk@n zHW+WMtQRM_z<_GaR^6ny$~0L=2I6%5bbsRY-WJkrBvFDGt)M~^dlF{R>uVn81v%(9 z0xHaKomKemK7A_~TefU&5fi4~F45(D7`f%>J+^4f%oEd31_6JU{xD}vEgb#JBwybT zE}s1*r#u&HPy>n7+8-0-%N;Kt%TB@}M=~Z%I9UGDUB)Mf+86~&bT2viQcsyl(4((* zDO!*TVP?(na*nEE8aEUJNt>x%nv~)ppUXvx0!c;JHj2vM)({`pt{{Q2W4lXEYWaTg zAz9eHot`LsQZH44fZwL9L$h4|Dc0n zhU!Y<@zSgdhh6#P%yvtbT^Jd|Ns8 zb$tJ&;^jw^{Hfp-RQRyRb zeq9=t80j|jkc=qZ@S;$6&d?c1&YTQrb|ox1>O=bwEyJZn_Z{Y45Nq@JZ&}2nXEK|3 zCbl~mGm{qe_jCkC$3@?5TI9|2#POe8!f8KYnyp=|4ywZ)M8a6U58dDp6fw7i-nGV{ux0JgIH)Fk|V zAw7AWYua+zWf=rWlst0Bt1R_C_p=ts=YBe593X7QiaLyw#qmBA;r$wn-xhAQ$C#-o zQlfY&!8>;{!x19)O=JHfiHWf?6NbE4R+eI0`QZa0Um=@)C)gJR8 ze0$I;>%ux@M|k&r8)?I?I-q!~Ea&HDK3g}o@N)n+(qc8;O=qj3o~b{eUFW5X;7Ig7 zndKO@gek$WZTh(x>hdjOY(1PB#Fw37z6+UtO-h zX#)G86P^ou2|Xds{%W~(MwpOuAi2gGpg-}r+Er4Kl+C+QQ@*?vMaoB7EVo40Ub-z!GjeWfB@q&h%xHUr@9$5_1)bak^a#>==o9@|AtbcVo@Lc zKb#a2^PQQpIs21ui=485n%K|$Il4%Y)?XtXuEfsYL)7n)4(EF7PY#bJy~WGV&-Nz( zg87Kl_JWzrP@cFwAzXVJWZW0ApH_;!~fAN3&|8OG5JA;4ntNhDf@AF@O3j5Mu=L7$P6CwY%zvJcK{@?lk_$~eq@QA+< literal 0 HcmV?d00001 diff --git a/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py b/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py index 953f028af3c..cf53fe90216 100644 --- a/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py +++ b/tests/unit/endpoints/speech/speech_to_completion_bridge/test_transformation.py @@ -11,14 +11,14 @@ from litellm.endpoints.speech.speech_to_completion_bridge.transformation import ) from litellm.types.utils import ChatCompletionAudioResponse, Choices, Message, ModelResponse +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + GEMINI_TTS_MODEL: Final = "gemini-3.1-flash-tts-preview" PCM_BYTES: Final = b"\x01\x02\x03\x04" * 6 def _model_response(model: str, pcm: bytes) -> ModelResponse: - audio: Final = ChatCompletionAudioResponse( - data=base64.b64encode(pcm).decode(), expires_at=0, transcript="hello" - ) + audio: Final = ChatCompletionAudioResponse(data=base64.b64encode(pcm).decode(), expires_at=0, transcript="hello") return ModelResponse(model=model, choices=[Choices(message=Message(content=None, audio=audio))]) @@ -72,6 +72,103 @@ def test_non_gemini_request_forwards_speech_response_format_as_audio_format() -> assert request["audio"] == {"voice": "alloy", "format": "wav"} +def test_gemini_request_keeps_voice_beside_nested_speech_config() -> None: + request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request( + model=GEMINI_TTS_MODEL, + input="Bonjour", + voice={"speech_config": {"language_code": "fr-FR"}, "voice": "Kore"}, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="gemini", + ) + + assert request["audio"]["speech_config"] == { + "languageCode": "fr-FR", + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}, + } + + +def test_gemini_request_keeps_voice_object_for_a_model_alias_map_alias(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "model_alias_map", {"voice-alias": GEMINI_TTS_MODEL}) + + request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request( + model="voice-alias", + input="Hello there", + voice={"voice": "Puck"}, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="gemini", + ) + + assert request["audio"] == { + "speech_config": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Puck"}}}, + "format": "pcm16", + } + + +@pytest.mark.parametrize("wrapper", (None, "speech_config", "speechConfig")) +def test_gemini_request_keeps_voice_and_language_without_audio_format(wrapper: str | None) -> None: + voice: Final = {"language_code": "fr-FR", "voice": "Kore", "format": "pcm16"} + request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request( + model=GEMINI_TTS_MODEL, + input="Bonjour", + voice=voice if wrapper is None else {wrapper: voice}, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="gemini", + ) + + assert request["audio"]["speech_config"] == { + "languageCode": "fr-FR", + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}, + } + + +@pytest.mark.parametrize( + ("drop_params", "voice_config"), + [ + (True, {"prebuiltVoiceConfig": {"voiceName": "Kore"}}), + (False, {"prebuiltVoiceConfig": {"voiceName": "Kore", "style": "calm"}, "tone": "warm"}), + ], +) +def test_gemini_request_drops_unknown_nested_voice_fields_only_with_drop_params( + drop_params: bool, voice_config: dict[str, object] +) -> None: + request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request( + model=GEMINI_TTS_MODEL, + input="Bonjour", + voice={"voice_config": {"prebuilt_voice_config": {"voice_name": "Kore", "style": "calm"}, "tone": "warm"}}, + optional_params={}, + litellm_params={"drop_params": drop_params}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="gemini", + ) + + assert request["audio"]["speech_config"] == {"voiceConfig": voice_config} + + +def test_gemini_request_reads_snake_case_wrapper_beside_null_camel_case_wrapper() -> None: + request: Final = SpeechToCompletionBridgeTransformationHandler().transform_request( + model=GEMINI_TTS_MODEL, + input="Bonjour", + voice={"speechConfig": None, "speech_config": {"voice_name": "Kore"}}, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="gemini", + ) + + assert request["audio"]["speech_config"] == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}} + + @pytest.mark.parametrize("response_format", ["mp3", "flac", "opus", "aac"]) def test_gemini_tts_request_rejects_formats_gemini_cannot_produce(response_format: str) -> None: with pytest.raises(litellm.BadRequestError) as excinfo: @@ -86,6 +183,7 @@ def test_gemini_tts_request_rejects_formats_gemini_cannot_produce(response_forma def test_gemini_tts_pcm_response_returns_raw_pcm_bytes() -> None: response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES), + model=GEMINI_TTS_MODEL, response_format="pcm", ) @@ -97,6 +195,7 @@ def test_gemini_tts_pcm_response_returns_raw_pcm_bytes() -> None: def test_gemini_tts_wav_and_default_responses_wrap_pcm_in_wav(response_format: str | None) -> None: response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( model_response=_model_response(GEMINI_TTS_MODEL, PCM_BYTES), + model=GEMINI_TTS_MODEL, response_format=response_format, ) @@ -110,8 +209,24 @@ def test_gemini_tts_wav_and_default_responses_wrap_pcm_in_wav(response_format: s def test_non_gemini_response_keeps_original_bytes_and_mpeg_content_type() -> None: response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( model_response=_model_response("gpt-4o-audio-preview", PCM_BYTES), + model="gpt-4o-audio-preview", response_format="mp3", ) assert response.response.content == PCM_BYTES assert response.response.headers["content-type"] == "audio/mpeg" + + +@pytest.mark.parametrize( + ("requested_model", "served_model"), + [(GEMINI_TTS_MODEL, "served-speech-alias"), ("requested-speech-alias", GEMINI_TTS_MODEL)], +) +def test_gemini_response_wraps_wav_when_either_model_name_is_gemini_tts(requested_model: str, served_model: str) -> None: + response: Final = SpeechToCompletionBridgeTransformationHandler().transform_response( + model_response=_model_response(served_model, PCM_BYTES), + model=requested_model, + response_format=None, + ) + + assert response.response.content[:4] == b"RIFF" + assert response.response.headers["content-type"] == "audio/wav" diff --git a/tests/unit/llms/gemini/test_gemini_tts.py b/tests/unit/llms/gemini/test_gemini_tts.py index a7522f6b315..d0ef04e6372 100644 --- a/tests/unit/llms/gemini/test_gemini_tts.py +++ b/tests/unit/llms/gemini/test_gemini_tts.py @@ -2,14 +2,70 @@ Test Gemini TTS (Text-to-Speech) functionality """ -import pytest +from typing import Final from unittest.mock import patch, MagicMock +import pytest import litellm +from litellm.endpoints.speech.speech_to_completion_bridge.transformation import ( + SpeechToCompletionBridgeTransformationHandler, +) from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.utils import get_supported_openai_params +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + +GEMINI_3_1_FLASH_TTS_MODEL = "gemini-3.1-flash-tts-preview" + +MULTI_SPEAKER_SPEECH_CONFIG = { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "Ryan", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Leda", + }, + }, + }, + ], + }, + "language_code": "en-US", +} + +NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG = { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speaker": "Ryan", + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voiceConfig": { + "prebuiltVoiceConfig": { + "voiceName": "Leda", + }, + }, + }, + ], + }, + "languageCode": "en-US", +} + class TestGeminiTTSTransformation: """Test Gemini TTS transformation functionality""" @@ -19,17 +75,16 @@ class TestGeminiTTSTransformation: config = GoogleAIStudioGeminiConfig() # Test TTS models (both preview and non-preview versions) - assert ( - config.is_model_gemini_audio_model("gemini-2.5-flash-preview-tts") == True - ) - assert config.is_model_gemini_audio_model("gemini-2.5-pro-preview-tts") == True - assert config.is_model_gemini_audio_model("gemini-2.5-flash-tts") == True - assert config.is_model_gemini_audio_model("gemini-2.5-pro-tts") == True + assert config.is_model_gemini_audio_model("gemini-2.5-flash-preview-tts") + assert config.is_model_gemini_audio_model("gemini-2.5-pro-preview-tts") + assert config.is_model_gemini_audio_model("gemini-2.5-flash-tts") + assert config.is_model_gemini_audio_model("gemini-2.5-pro-tts") + assert config.is_model_gemini_audio_model(GEMINI_3_1_FLASH_TTS_MODEL) # Test non-TTS models - assert config.is_model_gemini_audio_model("gemini-2.5-flash") == False - assert config.is_model_gemini_audio_model("gemini-2.5-pro") == False - assert config.is_model_gemini_audio_model("gpt-4o-audio-preview") == False + assert not config.is_model_gemini_audio_model("gemini-2.5-flash") + assert not config.is_model_gemini_audio_model("gemini-2.5-pro") + assert not config.is_model_gemini_audio_model("gpt-4o-audio-preview") def test_gemini_tts_supported_params(self): """Test that audio parameter is included for TTS models""" @@ -66,10 +121,7 @@ class TestGeminiTTSTransformation: assert "speechConfig" in result assert "voiceConfig" in result["speechConfig"] assert "prebuiltVoiceConfig" in result["speechConfig"]["voiceConfig"] - assert ( - result["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] - == "Kore" - ) + assert result["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" # Check response modalities assert "responseModalities" in result @@ -78,9 +130,7 @@ class TestGeminiTTSTransformation: def test_gemini_tts_audio_parameter_mapping_with_language_code(self): config = GoogleAIStudioGeminiConfig() - non_default_params = { - "audio": {"voice": "Kore", "format": "pcm16", "language_code": "en-US"} - } + non_default_params = {"audio": {"voice": "Kore", "format": "pcm16", "language_code": "en-US"}} optional_params = {} result = config.map_openai_params( @@ -102,6 +152,16 @@ class TestGeminiTTSTransformation: assert result["languageCode"] == "de-DE" assert result["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + def test_map_audio_params_keeps_voice_beside_nested_speech_config(self): + config = GoogleAIStudioGeminiConfig() + + result = config._map_audio_params( + {"speech_config": {"language_code": "fr-FR"}, "voice": "Kore", "format": "pcm16"} + ) + + assert result["languageCode"] == "fr-FR" + assert result["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + def test_map_audio_params_no_language_code(self): config = GoogleAIStudioGeminiConfig() @@ -110,6 +170,55 @@ class TestGeminiTTSTransformation: assert "languageCode" not in result assert result["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" + def test_map_audio_params_keeps_nested_speaker_list_and_drops_unknown_fields(self): + config: Final = GoogleAIStudioGeminiConfig() + + result: Final = config._map_audio_params( + { + "speech_config": { + "language_code": "fr-FR", + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + {"speaker": "Ryan", "voice_config": {"prebuilt_voice_config": {"voice_name": "Umbriel"}}} + ] + }, + "metadata": ["studio", 2], + }, + "format": "pcm16", + } + ) + + assert result == { + "languageCode": "fr-FR", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speaker": "Ryan", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Umbriel"}}} + ] + }, + } + + def test_gemini_tts_multi_speaker_audio_parameter_mapping(self): + """Test multi-speaker audio parameter mapping for Gemini 3.1 TTS models""" + config = GoogleAIStudioGeminiConfig() + + non_default_params = { + "audio": { + "speech_config": MULTI_SPEAKER_SPEECH_CONFIG, + "format": "pcm16", + } + } + optional_params = {} + + result = config.map_openai_params( + non_default_params=non_default_params, + optional_params=optional_params, + model=GEMINI_3_1_FLASH_TTS_MODEL, + drop_params=False, + ) + + assert result["speechConfig"] == NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG + assert result["responseModalities"] == ["AUDIO"] + def test_gemini_tts_audio_parameter_with_existing_modalities(self): """Test audio parameter mapping when modalities already exist""" config = GoogleAIStudioGeminiConfig() @@ -193,9 +302,7 @@ class TestGeminiTTSTransformation: } optional_params = {} - with pytest.raises( - ValueError, match="Unsupported audio format for Gemini TTS models" - ): + with pytest.raises(ValueError, match="Unsupported audio format for Gemini TTS models"): config.map_openai_params( non_default_params=non_default_params, optional_params=optional_params, @@ -234,6 +341,28 @@ def test_gemini_tts_completion_mock(): assert response.choices[0].message.content is not None +def test_gemini_tts_speech_bridge_accepts_multi_speaker_voice_dict(): + handler = SpeechToCompletionBridgeTransformationHandler() + + result = handler.transform_request( + model=f"vertex_ai/{GEMINI_3_1_FLASH_TTS_MODEL}", + input="Ryan: How are you doing today Katie?\nKatie: Not too bad.", + voice=MULTI_SPEAKER_SPEECH_CONFIG, + optional_params={"response_format": "pcm"}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="vertex_ai", + ) + + assert result["modalities"] == ["audio"] + assert result["audio"] == { + "speech_config": NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG, + "format": "pcm16", + } + assert "response_format" not in result + + class TestGeminiTTSSpeechConfigInRequestBody: """Test that speechConfig is properly included in the final request body. @@ -250,6 +379,8 @@ class TestGeminiTTSSpeechConfigInRequestBody: ("gemini-2.5-flash-preview-tts", "vertex_ai"), ("gemini-2.5-flash-preview-tts", "gemini"), ("gemini-2.5-pro-tts", "vertex_ai"), + (GEMINI_3_1_FLASH_TTS_MODEL, "vertex_ai"), + (GEMINI_3_1_FLASH_TTS_MODEL, "gemini"), ], ) def test_speechconfig_in_generation_config_transform_request_body(self, model, custom_llm_provider): @@ -283,12 +414,7 @@ class TestGeminiTTSSpeechConfigInRequestBody: f"speechConfig was filtered out of generationConfig for model={model}, provider={custom_llm_provider}. " "Ensure speechConfig is in the GenerationConfig TypedDict." ) - assert ( - generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"][ - "voiceName" - ] - == "Kore" - ) + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Kore" @pytest.mark.parametrize( "model,custom_llm_provider", @@ -296,6 +422,8 @@ class TestGeminiTTSSpeechConfigInRequestBody: ("gemini-2.5-flash-tts", "vertex_ai"), ("gemini-2.5-flash-tts", "gemini"), ("gemini-2.5-flash-preview-tts", "vertex_ai"), + (GEMINI_3_1_FLASH_TTS_MODEL, "vertex_ai"), + (GEMINI_3_1_FLASH_TTS_MODEL, "gemini"), ], ) def test_speechconfig_end_to_end_mapping(self, model, custom_llm_provider): @@ -342,17 +470,54 @@ class TestGeminiTTSSpeechConfigInRequestBody: f"speechConfig was filtered out during transform_request_body() for model={model}, provider={custom_llm_provider}. " "This breaks Gemini TTS - speechConfig must be in GenerationConfig TypedDict." ) - assert ( - generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"][ - "voiceName" - ] - == "Puck" - ) + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Puck" # Also verify responseModalities is present assert "responseModalities" in generation_config assert "AUDIO" in generation_config["responseModalities"] + @pytest.mark.parametrize( + "custom_llm_provider", + [ + "vertex_ai", + "gemini", + ], + ) + def test_multi_speaker_speechconfig_end_to_end_mapping(self, custom_llm_provider): + """Test full pipeline for Gemini 3.1 multi-speaker TTS speechConfig""" + from litellm.llms.vertex_ai.gemini.transformation import ( + _transform_request_body, + ) + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + config = VertexGeminiConfig() + + mapped_params = config.map_openai_params( + non_default_params={ + "audio": { + "speech_config": MULTI_SPEAKER_SPEECH_CONFIG, + "format": "pcm16", + } + }, + optional_params={}, + model=GEMINI_3_1_FLASH_TTS_MODEL, + drop_params=False, + ) + + request_body = _transform_request_body( + messages=[{"role": "user", "content": "Ryan: Hi.\nKatie: Hello."}], + model=GEMINI_3_1_FLASH_TTS_MODEL, + optional_params=mapped_params, + custom_llm_provider=custom_llm_provider, + litellm_params={}, + cached_content=None, + ) + + generation_config = request_body["generationConfig"] + assert generation_config["speechConfig"] == NORMALIZED_MULTI_SPEAKER_SPEECH_CONFIG + assert generation_config["responseModalities"] == ["AUDIO"] @pytest.mark.parametrize( "model,custom_llm_provider", @@ -395,14 +560,35 @@ class TestGeminiTTSSpeechConfigInRequestBody: generation_config = request_body["generationConfig"] assert generation_config["speechConfig"]["languageCode"] == "pt-BR" - assert ( - generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"][ - "voiceName" - ] - == "Puck" - ) + assert generation_config["speechConfig"]["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"] == "Puck" assert "AUDIO" in generation_config["responseModalities"] if __name__ == "__main__": pytest.main([__file__]) + + +@pytest.mark.parametrize("voice_key", ["name", "voiceName", "voice_name", "voice"]) +@pytest.mark.parametrize("wrapper", [None, "speechConfig", "speech_config"]) +@pytest.mark.parametrize("model", ["gemini-2.5-flash-tts", "gemini-2.5-pro-tts", GEMINI_3_1_FLASH_TTS_MODEL]) +def test_speech_bridge_preserves_single_speaker_voice_mapping(voice_key: str, wrapper: str | None, model: str): + handler = SpeechToCompletionBridgeTransformationHandler() + config = {voice_key: "Umbriel", "language_code": "en-US"} + voice = {wrapper: config} if wrapper else config + result = handler.transform_request( + model=model, + input="Hello", + voice=voice, + optional_params={}, + litellm_params={}, + headers={}, + litellm_logging_obj=MagicMock(), + custom_llm_provider="vertex_ai", + ) + assert result["audio"] == { + "speech_config": { + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Umbriel"}}, + "languageCode": "en-US", + }, + "format": "pcm16", + } diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_streaming.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_streaming.py index f59c584eb90..2a550b4a701 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_streaming.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_streaming.py @@ -278,6 +278,46 @@ class TestGetObjectNameLazyParse: assert exc_info.value.message.startswith("Batch input file line 1 could not be converted") +def _tts_batch_row(**body_extra: object) -> bytes: + body: Final = { + "model": "gemini-2.5-flash-preview-tts", + "messages": [{"role": "user", "content": "Say hi"}], + "audio": {"format": "pcm16", "speech_config": {"voice_name": "Kore", "style": "calm"}}, + **body_extra, + } + return json.dumps({"custom_id": "r-0", "method": "POST", "url": "/v1/chat/completions", "body": body}).encode() + + +def _upload_tts_batch_row(row: bytes, litellm_params: dict[str, object]) -> bytes: + request: Final[CreateFileRequest] = {"file": ("batch.jsonl", row + b"\n", "application/jsonl"), "purpose": "batch"} + return _join_upload_body( + VertexAIFilesConfig().transform_create_file_request( + model="", create_file_data=request, optional_params={}, litellm_params=litellm_params + ) + ) + + +class TestBatchRowUnknownVoiceKeys: + def test_unknown_voice_keys_are_a_400_without_drop_params(self) -> None: + with pytest.raises(VertexAIError, match="voice keys style") as exc_info: + _upload_tts_batch_row(_tts_batch_row(), {}) + assert exc_info.value.status_code == 400 + + @pytest.mark.parametrize( + "row, litellm_params", + [(_tts_batch_row(drop_params=True), {}), (_tts_batch_row(), {"drop_params": True})], + ids=["row", "deployment"], + ) + def test_drop_params_on_the_row_or_deployment_drops_unknown_voice_keys( + self, row: bytes, litellm_params: dict[str, object] + ) -> None: + uploaded: Final = json.loads(_upload_tts_batch_row(row, litellm_params)) + + assert uploaded["request"]["generationConfig"]["speechConfig"] == { + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}} + } + + class TestStreamingLaziness: def test_upload_stream_transforms_rows_only_as_they_are_pulled(self) -> None: cfg: Final = VertexAIFilesConfig() diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 6ea3039edd8..d2b5c8d5bd9 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -24,8 +24,11 @@ from litellm.llms.gemini.chat.transformation import GoogleAIStudioGeminiConfig from litellm.llms.vertex_ai.common_utils import GEMINI_MAX_ENUM_CHARACTERS, VertexAIError from litellm.llms.vertex_ai.vertex_llm_base import VertexBase from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + OPENAI_TO_GEMINI_TTS_VOICES, VertexGeminiConfig, _log_gemini_sampling_params_deprecated, + normalize_gemini_speech_config, + reject_unknown_gemini_tts_voice_keys, ) from litellm.types.llms.vertex_ai import GeminiFinishReason, RequestBody, UsageMetadata from litellm.types.utils import ChoiceLogprobs, Usage @@ -104,6 +107,242 @@ GEMINI_3_IMAGE_SIZE_MAPPINGS: Final[tuple[tuple[str, str, str], ...]] = ( ) +@pytest.mark.parametrize( + "speech_config", + ( + {"name": "aLlOy"}, + {"speech_config": {"voice_name": "aLlOy"}}, + {"voice_config": {"prebuilt_voice_config": {"voice_name": "aLlOy"}}}, + ), +) +def test_gemini_structured_speech_config_maps_openai_voice_names(speech_config: Mapping[str, object]) -> None: + assert normalize_gemini_speech_config(speech_config) == { + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES["alloy"]}} + } + + +@pytest.mark.parametrize( + "speaker", + ( + {"speakerAlias": "Ryan", "speakerId": "alloy"}, + {"speaker_alias": "Ryan", "speaker_id": "alloy"}, + ), +) +def test_gemini_speech_config_accepts_cloud_tts_speaker_entries(speaker: Mapping[str, object]) -> None: + speech_config: Final = { + "multi_speaker_voice_config": { + "speaker_voice_configs": [speaker, {"speaker": "Katie", "voice_config": {"voice_name": "Leda"}}] + } + } + + assert normalize_gemini_speech_config(speech_config) == { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speaker": "Ryan", + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES["alloy"]}}, + }, + {"speaker": "Katie", "voiceConfig": {"voiceName": "Leda"}}, + ] + } + } + + +@pytest.mark.parametrize( + ("voice", "expected"), + ( + ( + { + "languageCode": "en-US", + "modelName": "gemini-2.5-flash-tts", + "multiSpeakerVoiceConfig": {"speakerVoiceConfigs": [{"speakerAlias": "Ryan", "speakerId": "Umbriel"}]}, + }, + { + "languageCode": "en-US", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speaker": "Ryan", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Umbriel"}}} + ] + }, + }, + ), + ( + {"model_name": "gemini-2.5-flash-tts", "voice_config": {"prebuilt_voice_config": {"voice_name": "Leda"}}}, + {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Leda"}}}, + ), + ({"speech_config": {"modelName": "gemini-2.5-flash-tts", "language_code": "fr-FR"}}, {"languageCode": "fr-FR"}), + ), +) +def test_gemini_speech_config_drops_cloud_tts_model_name( + voice: Mapping[str, object], expected: Mapping[str, object] +) -> None: + assert normalize_gemini_speech_config(voice) == expected + + +def test_gemini_speech_config_drops_non_string_keys_without_adding_voice(): + speech_config: Final = { + "speech_config": { + "language_code": "fr-FR", + "voice_config": {"prebuilt_voice_config": {"voice_name": "Leda", 1: "discard"}}, + }, + "format": "pcm16", + } + + assert normalize_gemini_speech_config(speech_config) == { + "languageCode": "fr-FR", + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Leda"}}, + } + + +def test_gemini_speech_config_keeps_only_speech_config_fields(): + speech_config: Final = { + "voice": "Kore", + "metadata": {"source": "studio"}, + "speech_config": {"language_code": "fr-FR", "style": "calm"}, + } + + assert normalize_gemini_speech_config(speech_config) == { + "languageCode": "fr-FR", + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}, + } + + +@pytest.mark.parametrize( + "voice, unknown_keys", + [ + ({"name": "Kore", "style": "calm"}, "style"), + ({"speech_config": {"voice": "Kore", "pitch": 2}}, "pitch"), + ({"speechConfig": {"voiceConfig": {}}, "tone": "x", "pitch": 2}, "pitch, tone"), + ({"speechConfig": {"voiceName": "Kore"}, "speech_config": {"style": "calm"}}, "speech_config"), + ], +) +def test_unknown_gemini_tts_voice_keys_are_rejected_without_drop_params(voice, unknown_keys): + with pytest.raises(litellm.UnsupportedParamsError, match=f"voice keys {unknown_keys}\\."): + reject_unknown_gemini_tts_voice_keys("gemini-3.1-flash-tts-preview", voice, False, "vertex_ai") + + reject_unknown_gemini_tts_voice_keys("gemini-3.1-flash-tts-preview", voice, True, "vertex_ai") + + +def test_chat_audio_speech_config_unknown_keys_are_rejected_unless_dropped(): + audio: Final = {"format": "pcm16", "speech_config": {"voice_name": "Kore", "style": "calm"}} + + with pytest.raises(litellm.UnsupportedParamsError, match="voice keys style\\."): + VertexGeminiConfig().map_openai_params( + non_default_params={"audio": audio}, + optional_params={}, + model="gemini-2.5-flash-preview-tts", + drop_params=False, + ) + + dropped: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"audio": audio}, + optional_params={}, + model="gemini-2.5-flash-preview-tts", + drop_params=True, + ) + assert dropped["speechConfig"] == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}} + + voice_only: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"audio": {"voice": "Kore", "format": "pcm16", "style": "calm"}}, + optional_params={}, + model="gemini-2.5-flash-preview-tts", + drop_params=False, + ) + assert voice_only["speechConfig"] == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}} + + +def test_chat_audio_speech_config_unknown_keys_follow_global_drop_params(monkeypatch): + monkeypatch.setattr(litellm, "drop_params", True) + + dropped: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"audio": {"format": "pcm16", "speech_config": {"voice_name": "Kore", "style": "calm"}}}, + optional_params={}, + model="gemini-2.5-flash-preview-tts", + drop_params=False, + ) + + assert dropped["speechConfig"] == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}} + + +def _chat_speech_config(speech_config: Mapping[str, object], drop_params: bool) -> object: + return VertexGeminiConfig().map_openai_params( + non_default_params={"audio": {"format": "pcm16", "speech_config": speech_config}}, + optional_params={}, + model="gemini-2.5-flash-preview-tts", + drop_params=drop_params, + )["speechConfig"] + + +def test_chat_audio_drop_params_strips_unknown_fields_inside_speaker_voices(): + speech_config: Final = { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + {"speaker": "Joe", "voice_config": {"prebuilt_voice_config": {"voice_name": "Puck", "pitch": 2}}}, + {"speaker_alias": "Jane", "speaker_id": "Kore", "style": "calm"}, + ], + "turn_order": "alternate", + } + } + + assert _chat_speech_config(speech_config, drop_params=True) == { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speaker": "Joe", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Puck"}}}, + {"speaker": "Jane", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}}, + ] + } + } + + +GEMINI_VOICE_CONFIG: Final = { + "voice": "Kore", + "prebuiltVoiceConfig": {"voiceName": "Kore"}, + "replicatedVoiceConfig": {"mimeType": "audio/wav", "voiceSampleAudio": "UklGRg=="}, +} + + +@pytest.mark.parametrize( + "speech_config", + [ + {"voiceConfig": GEMINI_VOICE_CONFIG, "languageCode": "fr-FR"}, + { + "voice_config": { + "voice": "Kore", + "prebuilt_voice_config": {"voice_name": "Kore"}, + "replicated_voice_config": {"mime_type": "audio/wav", "voice_sample_audio": "UklGRg=="}, + }, + "language_code": "fr-FR", + }, + ], +) +def test_chat_audio_drop_params_keeps_every_voice_config_field_gemini_accepts(speech_config: dict[str, object]): + assert _chat_speech_config(speech_config, drop_params=True) == { + "voiceConfig": GEMINI_VOICE_CONFIG, + "languageCode": "fr-FR", + } + + +def test_chat_audio_reads_snake_case_wrapper_beside_null_camel_case_wrapper(): + mapped: Final = VertexGeminiConfig().map_openai_params( + non_default_params={"audio": {"format": "pcm16", "speechConfig": None, "speech_config": {"voice_name": "Kore"}}}, + optional_params={}, + model="gemini-2.5-flash-preview-tts", + drop_params=False, + ) + + assert mapped["speechConfig"] == {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}} + + +def test_unknown_voice_key_error_stays_bounded_for_large_voice_objects(): + voice: Final = {"name": "Kore", **{f"{index:04d}{'x' * 1000}": index for index in range(2000)}} + + with pytest.raises(litellm.UnsupportedParamsError) as excinfo: + reject_unknown_gemini_tts_voice_keys("gemini-3.1-flash-tts-preview", voice, False, "vertex_ai") + + assert "and 1990 more" in excinfo.value.message + assert len(excinfo.value.message) < 1500 + + def test_top_logprobs(): non_default_params = { "top_logprobs": 2, diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index dc6f35fcfe5..b3d6d5b952f 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,4 +1,10 @@ import base64 +import io +import math +import struct +import sys +import wave +from pathlib import Path from typing import Final import json from collections.abc import Iterator, Mapping @@ -12,13 +18,61 @@ from pydantic import ValidationError import litellm from litellm.litellm_core_utils.exception_mapping_utils import exception_type +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES from litellm.llms.vertex_ai.text_to_speech.transformation import ( VertexAILyriaTextToSpeechConfig, VertexAITextToSpeechConfig, + _fallback_gemini_tts_audio_duration, ) +from litellm.cost_calculator import response_cost_calculator +from litellm.litellm_core_utils.audio_utils.utils import calculate_request_duration from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + + +@pytest.mark.parametrize( + ("voice", "multi_speaker"), + ( + ("aLlOy", False), + ({"name": "aLlOy"}, False), + ({"voice_config": {"prebuilt_voice_config": {"voice_name": "aLlOy"}}}, False), + ( + { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + {"speaker": "reader", "voice_config": {"prebuilt_voice_config": {"voice_name": "aLlOy"}}} + ] + } + }, + True, + ), + ), +) +def test_gemini_cloud_tts_maps_openai_voice_names_in_all_voice_forms( + voice: str | dict[str, object], multi_speaker: bool +) -> None: + config: Final = VertexAITextToSpeechConfig() + model: Final = "gemini-3.1-flash-tts-preview" + _, params = config.map_openai_params(model=model, voice=voice, optional_params={"response_format": "mp3"}) + mapped_voice: Final = OPENAI_TO_GEMINI_TTS_VOICES["alloy"] + expected_voice: Final = { + "languageCode": config.DEFAULT_LANGUAGE_CODE, + "modelName": model, + **( + { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [{"speakerAlias": "reader", "speakerId": mapped_voice}] + } + } + if multi_speaker + else {"name": mapped_voice} + ), + } + + assert params["vertex_voice_dict"] == expected_voice + class TestVertexAITextToSpeechConfig: """Tests for VertexAITextToSpeechConfig transformation""" @@ -48,8 +102,12 @@ class TestVertexAITextToSpeechConfig: assert url == custom_url - @patch.object(VertexAITextToSpeechConfig, "_ensure_access_token") - @patch.object(VertexAITextToSpeechConfig, "_get_token_and_url") + @patch.object( # test-quality-ok: isolates provider credentials while testing request serialization + VertexAITextToSpeechConfig, "_ensure_access_token" + ) + @patch.object( # test-quality-ok: fixes the provider URL at the authentication boundary + VertexAITextToSpeechConfig, "_get_token_and_url" + ) def test_transform_text_to_speech_request_body(self, mock_get_token, mock_ensure_token): """Test that transform_text_to_speech_request generates correct request body""" # Mock authentication @@ -94,6 +152,20 @@ class TestVertexAITextToSpeechConfig: assert "headers" in result assert "Authorization" in result["headers"] + raw_voice_request: Final = config.transform_text_to_speech_request( + model="vertex_ai/chirp", + input="Hello", + voice="en-US-Chirp3-HD-Charon", + optional_params={}, + litellm_params={ + "vertex_credentials": None, + "vertex_project": "test-project", + "vertex_location": "us-central1", + }, + headers={}, + ) + assert raw_voice_request["dict_body"]["voice"]["name"] == "en-US-Chirp3-HD-Charon" + def test_voice_mapping_openai_to_vertex(self): """Test that OpenAI voice names are correctly mapped to Vertex AI voices""" config = VertexAITextToSpeechConfig() @@ -131,6 +203,506 @@ class TestVertexAITextToSpeechConfig: assert voice_str is None assert voice_dict == voice_input + def test_gemini_tts_multi_speaker_voice_mapping(self): + config = VertexAITextToSpeechConfig() + + voice = { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "Ryan", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Umbriel", + }, + }, + }, + { + "speaker": "Katie", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Leda", + }, + }, + }, + ], + }, + } + + expected_voice = { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speakerAlias": "Ryan", + "speakerId": "Umbriel", + }, + { + "speakerAlias": "Katie", + "speakerId": "Leda", + }, + ], + }, + } + + voice_str, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + assert voice_str is None + assert optional_params["audioEncoding"] == "MP3" + assert optional_params["vertex_voice_dict"] == expected_voice + + voice_str, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "opus"}, + voice=voice, + ) + assert voice_str is None + assert optional_params["audioEncoding"] == "OGG_OPUS" + assert optional_params["vertex_voice_dict"] == expected_voice + + def test_gemini_tts_nested_language_and_voice_alias_reach_cloud_tts(self): + config: Final = VertexAITextToSpeechConfig() + voice: Final = {"speech_config": {"language_code": "fr-FR"}, "voice": "Kore"} + + voice_name, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + + assert voice_name is None + assert optional_params["vertex_voice_dict"] == { + "languageCode": "fr-FR", + "modelName": "gemini-3.1-flash-tts-preview", + "name": "Kore", + } + + @pytest.mark.parametrize( + ("voice", "expected_voice"), + [ + ( + { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + {"speaker": "Ryan", "voice_config": {"prebuilt_voice_config": {"voice_name": "Umbriel"}}}, + {"speaker": "Katie", "voice_config": {"prebuilt_voice_config": {"voice_name": "Leda"}}}, + ], + }, + "speech_config": {"language_code": "fr-FR"}, + }, + { + "languageCode": "fr-FR", + "modelName": "gemini-3.1-flash-tts-preview", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speakerAlias": "Ryan", "speakerId": "Umbriel"}, + {"speakerAlias": "Katie", "speakerId": "Leda"}, + ], + }, + }, + ), + ( + { + "voice_config": {"prebuilt_voice_config": {"voice_name": "Puck"}}, + "speechConfig": {"languageCode": "en-GB"}, + }, + {"languageCode": "en-GB", "modelName": "gemini-3.1-flash-tts-preview", "name": "Puck"}, + ), + ], + ) + def test_gemini_tts_voice_beside_speech_config_wrapper_reaches_cloud_tts( + self, voice: dict[str, object], expected_voice: dict[str, object] + ): + config: Final = VertexAITextToSpeechConfig() + + voice_name, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + + assert voice_name is None + assert optional_params["vertex_voice_dict"] == expected_voice + + @pytest.mark.parametrize( + ("voice", "expected_language", "expected_voices"), + [ + ( + { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speaker": "Ryan", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Umbriel"}}}, + {"speaker": "Katie", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Leda"}}}, + ], + }, + "speech_config": { + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + {"speaker": "Ana", "voice_config": {"prebuilt_voice_config": {"voice_name": "Kore"}}}, + {"speaker": "Ben", "voice_config": {"prebuilt_voice_config": {"voice_name": "Puck"}}}, + ], + }, + }, + }, + None, + (("Ana", "Kore"), ("Ben", "Puck")), + ), + ( + {"name": "Puck", "languageCode": "fr-FR", "speech_config": {"language_code": "de-DE"}}, + "de-DE", + ((None, "Puck"),), + ), + ( + {"name": "Puck", "voice_config": {"prebuilt_voice_config": {"voice_name": "Charon"}}}, + None, + ((None, "Charon"),), + ), + ], + ) + def test_gemini_tts_cloud_tts_speaks_the_voice_wav_sends_to_gemini( + self, + voice: dict[str, object], + expected_language: str | None, + expected_voices: tuple[tuple[str | None, str], ...], + ): + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import normalize_gemini_speech_config + + _, optional_params = VertexAITextToSpeechConfig().map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + cloud_tts_voice: Final = optional_params["vertex_voice_dict"] + cloud_tts_speakers: Final = cloud_tts_voice.get("multiSpeakerVoiceConfig", {}).get("speakerVoiceConfigs") + cloud_tts_voices: Final = ( + tuple((speaker["speakerAlias"], speaker["speakerId"]) for speaker in cloud_tts_speakers) + if cloud_tts_speakers is not None + else ((None, cloud_tts_voice["name"]),) + ) + wav_config: Final = normalize_gemini_speech_config(voice, drop_unknown_fields=True) + wav_speakers: Final = wav_config.get("multiSpeakerVoiceConfig", {}).get("speakerVoiceConfigs") + wav_voices: Final = ( + tuple( + (speaker["speaker"], speaker["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"]) + for speaker in wav_speakers + ) + if wav_speakers is not None + else ((None, wav_config["voiceConfig"]["prebuiltVoiceConfig"]["voiceName"]),) + ) + + assert cloud_tts_voices == wav_voices == expected_voices + assert cloud_tts_voice["languageCode"] == (expected_language or "en-US") + assert wav_config.get("languageCode") == expected_language + + def test_gemini_tts_string_voice_reaches_cloud_tts(self): + config: Final = VertexAITextToSpeechConfig() + + voice_name, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice="Kore", + ) + + assert voice_name == "Kore" + assert optional_params["vertex_voice_dict"] == { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "name": "Kore", + } + + @pytest.mark.parametrize( + "speaker_voice_configs", + ( + [None, {"speakerAlias": "Katie", "speakerId": "Leda"}], + [{"speakerAlias": "Ryan"}, {"speakerAlias": "Katie", "speakerId": "Leda"}], + [ + {"speaker": "Ryan", "voice_config": {"prebuilt_voice_config": {"name": "Umbriel"}}}, + {"speakerAlias": "Katie", "speakerId": "Leda"}, + ], + "invalid", + ), + ) + def test_gemini_tts_malformed_speaker_entry_is_a_bad_request(self, speaker_voice_configs: object): + config: Final = VertexAITextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError, match="speaker_voice_configs"): + config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice={"multiSpeakerVoiceConfig": {"speakerVoiceConfigs": speaker_voice_configs}}, + ) + + def test_gemini_tts_malformed_speaker_entry_is_dropped_with_drop_params(self): + config: Final = VertexAITextToSpeechConfig() + + _, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice={ + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speakerAlias": "Ryan", "speakerId": "Umbriel"}, + {"speakerAlias": "Katie"}, + ] + } + }, + drop_params=True, + ) + + assert optional_params["vertex_voice_dict"]["multiSpeakerVoiceConfig"] == { + "speakerVoiceConfigs": [{"speakerAlias": "Ryan", "speakerId": "Umbriel"}] + } + + @pytest.mark.parametrize( + "voice", + [ + {"speechConfig": {"voice": "Kore"}}, + {"speechConfig": {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}}}}, + {"speechConfig": {"voiceConfig": {}}}, + {"speech_config": {}}, + {"multi_speaker_voice_config": {"speaker_voice_configs": []}}, + {"languageCode": "en-US"}, + ], + ) + def test_gemini_tts_nested_voice_mapping_sends_only_cloud_tts_voice_fields(self, voice: dict[str, object]): + config: Final = VertexAITextToSpeechConfig() + + _, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + + assert optional_params["vertex_voice_dict"] == { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "name": "Kore", + } + + @pytest.mark.parametrize( + ("voice", "unsendable_field"), + [ + ( + {"voice_config": {"replicated_voice_config": {"mime_type": "audio/wav", "voice_sample_audio": "UklG"}}}, + "voiceConfig.replicatedVoiceConfig", + ), + ( + {"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Puck", "style": "calm"}}}, + "voiceConfig.prebuiltVoiceConfig.style", + ), + ({"speechConfig": {"voiceConfig": {"voice": "Puck"}}}, "voiceConfig.voice"), + ( + { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speaker": "Ryan", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Puck"}}}, + {"speaker": "Katie", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Leda"}}, "tone": "x"}, + ] + } + }, + "multiSpeakerVoiceConfig.speakerVoiceConfigs.tone", + ), + ( + { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speaker": "Ryan", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Puck"}}, "tone": "x"}, + {"speaker": "Katie", "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Leda"}}, "tone": "y"}, + ] + } + }, + "multiSpeakerVoiceConfig.speakerVoiceConfigs.tone", + ), + ( + { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speakerAlias": "Ryan", + "speakerId": "Puck", + "voiceConfig": { + "replicatedVoiceConfig": {"mimeType": "audio/wav", "voiceSampleAudio": "AAAA"} + }, + }, + {"speakerAlias": "Katie", "speakerId": "Kore"}, + ] + } + }, + "multiSpeakerVoiceConfig.speakerVoiceConfigs.voiceConfig.replicatedVoiceConfig", + ), + ({"languageCode": ["en-US"], "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Puck"}}}, "languageCode"), + ({"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": 5}}}, "voiceConfig.prebuiltVoiceConfig.voiceName"), + ({"voiceConfig": "Puck"}, "voiceConfig"), + ({"multiSpeakerVoiceConfig": [{"speaker": "Ryan"}]}, "multiSpeakerVoiceConfig"), + ({"name": 5}, "name"), + ({"speechConfig": {"voice": ["Puck"]}}, "speechConfig.voice"), + ({"speechConfig": "Puck"}, "speechConfig"), + ({"speech_config": [{"voice_config": {"prebuilt_voice_config": {"voice_name": "Puck"}}}]}, "speech_config"), + ( + { + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + { + "speakerAlias": "Ryan", + "speakerId": "Umbriel", + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Puck"}}, + }, + {"speakerAlias": "Katie", "speakerId": "Kore"}, + ] + } + }, + "multiSpeakerVoiceConfig.speakerVoiceConfigs.voiceConfig.prebuiltVoiceConfig", + ), + ], + ) + def test_gemini_tts_cloud_tts_rejects_voice_fields_it_cannot_send( + self, voice: dict[str, object], unsendable_field: str + ): + config: Final = VertexAITextToSpeechConfig() + + with pytest.raises(litellm.UnsupportedParamsError) as exc_info: + config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + ) + + assert f"voice fields {unsendable_field} as mp3" in str(exc_info.value) + + @pytest.mark.parametrize( + ("voice", "voice_name"), + [ + ({"voiceConfig": {"replicatedVoiceConfig": {"mimeType": "audio/wav", "voiceSampleAudio": "UklG"}}}, "Kore"), + ({"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Puck", "style": "calm"}}}, "Puck"), + ({"voiceConfig": "Puck"}, "Kore"), + ({"speechConfig": "Puck"}, "Kore"), + ], + ) + def test_gemini_tts_cloud_tts_drops_voice_fields_it_cannot_send_with_drop_params( + self, voice: dict[str, object], voice_name: str + ): + config: Final = VertexAITextToSpeechConfig() + + _, optional_params = config.map_openai_params( + model="gemini-3.1-flash-tts-preview", + optional_params={"response_format": "mp3"}, + voice=voice, + drop_params=True, + ) + + assert optional_params["vertex_voice_dict"] == { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "name": voice_name, + } + + @pytest.mark.parametrize( + "voice", + [ + {"name": "Umbriel", "modelName": "gemini-2.5-flash-tts"}, + {"name": "Umbriel", "model_name": "gemini-2.5-flash-tts"}, + {"modelName": "gemini-2.5-flash-tts", "model_name": "chirp-3"}, + { + "modelName": "gemini-2.5-flash-tts", + "model_name": "chirp-3", + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + { + "speaker": "Ryan", + "voice_config": { + "prebuilt_voice_config": { + "voice_name": "Umbriel", + }, + }, + }, + ], + }, + }, + ], + ) + def test_gemini_tts_ignores_voice_model_name_override(self, voice): + config = VertexAITextToSpeechConfig() + routed_model = "gemini-3.1-flash-tts-preview" + + _, optional_params = config.map_openai_params( + model=routed_model, + optional_params={"response_format": "mp3"}, + voice=voice, + ) + assert optional_params["vertex_voice_dict"]["modelName"] == routed_model + + def test_dispatch_maps_gemini_cloud_tts_params_when_provider_config_skipped(self): + config = VertexAITextToSpeechConfig() + handler = MagicMock() + handler.text_to_speech_handler.return_value = "ok" + + result = config.dispatch_text_to_speech( + model="gemini-3.1-flash-tts-preview", + input="Hi", + voice={ + "name": "Umbriel", + "modelName": "gemini-2.5-flash-tts", + }, + optional_params={"response_format": "mp3"}, + litellm_params_dict={}, + logging_obj=MagicMock(), + timeout=10, + extra_headers=None, + base_llm_http_handler=handler, + aspeech=False, + api_base=None, + api_key=None, + ) + + assert result == "ok" + mapped_params = handler.text_to_speech_handler.call_args.kwargs["text_to_speech_optional_params"] + assert mapped_params["audioEncoding"] == "MP3" + assert mapped_params["vertex_voice_dict"]["name"] == "Umbriel" + assert mapped_params["vertex_voice_dict"]["modelName"] == "gemini-3.1-flash-tts-preview" + + @pytest.mark.parametrize( + ("voice", "expected_name"), + [ + ("en-US-Chirp3-HD-Charon", "en-US-Chirp3-HD-Charon"), + ({"name": "en-US-Chirp3-HD-Charon"}, "en-US-Chirp3-HD-Charon"), + (None, None), + ], + ) + def test_dispatch_keeps_pre_mapped_cloud_tts_params( + self, voice: str | dict[str, str] | None, expected_name: str | None + ): + config = VertexAITextToSpeechConfig() + handler = MagicMock() + handler.text_to_speech_handler.return_value = "ok" + optional_params = { + "audioEncoding": "OGG_OPUS", + "vertex_voice_dict": {"languageCode": "en-US", "name": "en-US-Chirp3-HD-Charon"}, + } + + config.dispatch_text_to_speech( + model="chirp", + input="Hi", + voice=voice, + optional_params=optional_params, + litellm_params_dict={}, + logging_obj=MagicMock(), + timeout=10, + extra_headers=None, + base_llm_http_handler=handler, + aspeech=False, + api_base=None, + api_key=None, + ) + + call_kwargs = handler.text_to_speech_handler.call_args.kwargs + assert call_kwargs["voice"] == expected_name + assert call_kwargs["text_to_speech_optional_params"] is optional_params + assert call_kwargs["text_to_speech_optional_params"]["audioEncoding"] == "OGG_OPUS" + @pytest.mark.parametrize( ("audio", "expected_content_type"), @@ -174,6 +746,150 @@ def test_transform_text_to_speech_response_leaves_unknown_bytes_unlabeled(): assert result.response.content == raw_pcm +@pytest.mark.parametrize("encoding", ["LINEAR16", "PCM", "MP3", "OGG_OPUS", "ALAW", "MULAW"]) +def test_gemini_cloud_tts_response_bills_text_and_audio(encoding: str, monkeypatch: pytest.MonkeyPatch) -> None: + model: Final = "gemini-3.1-flash-tts-preview" + input_text: Final = "Hello from Gemini text to speech" + pcm_audio: Final = b"\x00\x00" * 24000 + wav_buffer: Final = io.BytesIO() + with wave.open(wav_buffer, "wb") as wav_file: + wav_file.setnchannels(1) + wav_file.setsampwidth(2) + wav_file.setframerate(24000) + wav_file.writeframes(pcm_audio) + g711_audio: Final = ( + struct.pack( + "<4sI4s4sIHHIIHH4sI", + b"RIFF", + 36 + 24000, + b"WAVE", + b"fmt ", + 16, + 6 if encoding == "ALAW" else 7, + 1, + 24000, + 24000, + 1, + 8, + b"data", + 24000, + ) + + b"\x00" * 24000 + ) + fixture_dir: Final = Path(__file__).resolve().parents[4] / "audio_tests" + audio_by_encoding: Final = { + "PCM": pcm_audio, + "LINEAR16": wav_buffer.getvalue(), + "MP3": (fixture_dir / "gemini_tts_speech.mp3").read_bytes(), + "OGG_OPUS": (fixture_dir / "gemini_tts_speech.ogg").read_bytes(), + "ALAW": g711_audio, + "MULAW": g711_audio, + } + audio_bytes: Final = audio_by_encoding[encoding] + logger: Final = MagicMock() + sent_request: Final = httpx.Request( + "POST", + _SYNTHESIZE_URL, + json={ + "input": {"text": input_text}, + "audioConfig": {"audioEncoding": encoding, "sampleRateHertz": 24000}, + }, + ) + raw_response: Final = httpx.Response( + 200, json={"audioContent": base64.b64encode(audio_bytes).decode()}, request=sent_request + ) + + result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response(model, raw_response, logger) + usage: Final = result.usage + assert usage is not None + duration: Final = len(audio_bytes) / 48000 if encoding == "PCM" else calculate_request_duration(audio_bytes) + assert duration is not None + assert _fallback_gemini_tts_audio_duration(audio_bytes, encoding, 24000) == pytest.approx(duration), ( + "Google Cloud AudioEncoding and RFC 7845, checked 2026-09-23: " + "https://cloud.google.com/text-to-speech/docs/reference/rest/v1/AudioEncoding " + "https://www.rfc-editor.org/rfc/rfc7845.html" + ) + assert usage.completion_tokens == math.ceil(duration * 25), ( + "Google Cloud TTS pricing, 2026-09-23: https://cloud.google.com/text-to-speech/pricing" + ) + assert usage.prompt_tokens == litellm.token_counter(model=model, text=input_text) + assert usage.total_tokens == usage.prompt_tokens + usage.completion_tokens + assert usage.completion_tokens_details is not None + assert usage.completion_tokens_details.audio_tokens == usage.completion_tokens + with monkeypatch.context() as dependencies: + dependencies.setitem(sys.modules, "soundfile", None) + sdk_result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response(model, raw_response, logger) + assert sdk_result.usage == usage + + model_info: Final = litellm.get_model_info(model, custom_llm_provider="vertex_ai") + cost: Final = response_cost_calculator( + response_object=result, + model=model, + custom_llm_provider="vertex_ai", + call_type="speech", + optional_params={}, + prompt=input_text, + ) + expected_cost: Final = ( + usage.prompt_tokens * model_info["input_cost_per_token"] + + usage.completion_tokens * model_info["output_cost_per_audio_token"] + ) + assert cost == pytest.approx(expected_cost) + assert cost > usage.prompt_tokens * model_info["input_cost_per_token"] + + +@pytest.mark.parametrize("encoding", ["ALAW", "MULAW"]) +@pytest.mark.parametrize("audio", [b"\x12" * 24000, b"RIFF" + b"\x12" * 23996]) +def test_gemini_cloud_tts_returns_raw_g711_audio_when_duration_decoder_is_unavailable(encoding: str, audio: bytes): + logger: Final = MagicMock() + sent_request: Final = httpx.Request( + "POST", + _SYNTHESIZE_URL, + json={ + "input": {"text": "Hello"}, + "audioConfig": {"audioEncoding": encoding, "sampleRateHertz": 24000}, + }, + ) + raw_response: Final = httpx.Response( + 200, json={"audioContent": base64.b64encode(audio).decode()}, request=sent_request + ) + + with patch("litellm.llms.vertex_ai.text_to_speech.transformation.calculate_request_duration", return_value=None): + result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response( + "gemini-3.1-flash-tts-preview", raw_response, logger + ) + + assert result.response.content == audio + assert result.usage is not None + assert result.usage.prompt_tokens > 0 + assert result.usage.completion_tokens == 25, ( + "Google Cloud TTS pricing, 2026-09-23: https://cloud.google.com/text-to-speech/pricing" + ) + + +def test_gemini_cloud_tts_returns_audio_unbilled_when_duration_is_unmeasurable(): + logger: Final = MagicMock() + sent_request: Final = httpx.Request( + "POST", + _SYNTHESIZE_URL, + json={ + "input": {"text": "Hello"}, + "audioConfig": {"audioEncoding": "MP3", "sampleRateHertz": 24000}, + }, + ) + raw_response: Final = httpx.Response( + 200, json={"audioContent": base64.b64encode(b"invalid").decode()}, request=sent_request + ) + + with patch("litellm.llms.vertex_ai.text_to_speech.transformation.calculate_request_duration", return_value=None): + result: Final = VertexAITextToSpeechConfig().transform_text_to_speech_response( + "gemini-3.1-flash-tts-preview", raw_response, logger + ) + + assert result.response.content == b"invalid" + assert result.usage is None + + class TestVertexAILyriaTextToSpeechConfig: @pytest.mark.parametrize( "model", @@ -929,3 +1645,516 @@ def test_lyria_malformed_item_position_does_not_change_the_mapped_error(payload: custom_llm_provider="vertex_ai", original_exception=exc_info.value, ) + + +_PCM_AUDIO: Final = b"\x00\x00\x01\x00" * 2400 +_GENERATE_CONTENT_RESPONSE: Final = { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "inlineData": { + "mimeType": "audio/L16;codec=pcm;rate=24000", + "data": base64.b64encode(_PCM_AUDIO).decode(), + } + } + ], + }, + "finishReason": "STOP", + } + ], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 10, "totalTokenCount": 15}, +} + + +@pytest.mark.parametrize( + ("model", "response_format"), + ( + ("gemini-3.8-flash-tts", None), + ("gemini-3.1-flash-tts-preview", None), + ("gemini-3.1-flash-tts-preview", "wav"), + ("gemini-3.1-flash-tts-preview", "pcm"), + ), +) +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_formats_the_bridge_serves_stay_on_generate_content( + respx_mock: respx.MockRouter, + google_token_endpoint: responses.RequestsMock, + model: str, + response_format: str | None, +) -> None: + route: Final = respx_mock.post( + url__regex=rf"https://aiplatform\.googleapis\.com/.*/models/{model}:generateContent$" + ).mock(return_value=httpx.Response(200, json=_GENERATE_CONTENT_RESPONSE)) + + response: Final = await litellm.aspeech( + model=f"vertex_ai/{model}", + input="Say hello", + voice="Kore", + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + **({} if response_format is None else {"response_format": response_format}), + ) + + assert route.call_count == 1 + assert _PCM_AUDIO in response.content + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_reaches_cloud_tts_for_any_gemini_tts_model( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + + response: Final = await litellm.aspeech( + model="vertex_ai/gemini-3.8-flash-tts", + input="Say hello", + voice="Kore", + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + assert response.content == audio_bytes + sent: Final = json.loads(route.calls[0].request.content) + assert sent["voice"]["modelName"] == "gemini-3.8-flash-tts" + assert sent["voice"]["name"] == "Kore" + assert sent["audioConfig"]["audioEncoding"] == "MP3" + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_without_voice_keeps_the_gemini_model( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Say hello", + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + sent: Final = json.loads(route.calls[0].request.content) + assert sent["voice"] == {"languageCode": "en-US", "modelName": "gemini-3.1-flash-tts-preview", "name": "Kore"} + + +@pytest.mark.parametrize("voice", ("en-US-Studio-O", "en-US-Neural2-F")) +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_rejects_a_cloud_tts_voice( + voice: str, respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + with pytest.raises(litellm.BadRequestError, match=f"{voice} is a Cloud Text-to-Speech voice"): + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Say hello", + voice=voice, + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_drop_params_swaps_a_cloud_tts_voice_for_a_gemini_voice( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Say hello", + voice="en-US-Studio-O", + response_format="mp3", + drop_params=True, + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + assert json.loads(route.calls[0].request.content)["voice"]["name"] == "Kore" + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_without_a_location_uses_the_vertex_default_region( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.delenv("VERTEXAI_LOCATION", raising=False) + monkeypatch.delenv("VERTEX_LOCATION", raising=False) + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post("https://us-texttospeech.googleapis.com/v1/text:synthesize").mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Say hello", + voice="Kore", + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + ) + + assert route.called + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_sends_instructions_as_the_style_prompt( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Have a wonderful day", + voice="Kore", + response_format="mp3", + instructions="Read this slowly and calmly", + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + sent: Final = json.loads(route.calls[0].request.content) + assert sent["input"] == {"text": "Have a wonderful day", "prompt": "Read this slowly and calmly"} + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_with_a_numeric_speaking_rate_returns_billed_audio( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + + result: Final = await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Hello there", + voice="Kore", + response_format="mp3", + audioConfig={"audioEncoding": "MP3", "speakingRate": 1.25}, + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + assert json.loads(route.calls[0].request.content)["audioConfig"]["speakingRate"] == 1.25 + assert result.content == audio_bytes + assert result.usage is not None + assert result.usage.completion_tokens > 0 + + +@pytest.mark.parametrize( + ("vertex_location", "synthesize_url"), + ( + ("us-central1", "https://us-texttospeech.googleapis.com/v1/text:synthesize"), + ("us-east5", "https://us-texttospeech.googleapis.com/v1/text:synthesize"), + ("us", "https://us-texttospeech.googleapis.com/v1/text:synthesize"), + ("europe-west4", "https://eu-texttospeech.googleapis.com/v1/text:synthesize"), + ("europe-north1", "https://eu-texttospeech.googleapis.com/v1/text:synthesize"), + ("eu", "https://eu-texttospeech.googleapis.com/v1/text:synthesize"), + ("northamerica-northeast1", "https://northamerica-northeast1-texttospeech.googleapis.com/v1/text:synthesize"), + ("global", _SYNTHESIZE_URL), + ), +) +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_keeps_the_deployment_region( + respx_mock: respx.MockRouter, + google_token_endpoint: responses.RequestsMock, + vertex_location: str, + synthesize_url: str, +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(synthesize_url).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + + response: Final = await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Say hello", + voice="Kore", + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + vertex_location=vertex_location, + ) + + assert route.call_count == 1 + assert response.content == audio_bytes + + +@pytest.mark.parametrize("vertex_location", ["asia-northeast1", "europe-west2", "australia-southeast1"]) +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_rejects_a_region_cloud_tts_does_not_serve( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock, vertex_location: str +) -> None: + with pytest.raises(litellm.BadRequestError, match=f"no Gemini TTS location for vertex_location {vertex_location}"): + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Say hello", + voice="Kore", + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + vertex_location=vertex_location, + ) + + assert not respx_mock.calls + + +@pytest.mark.asyncio +async def test_router_gemini_tts_mp3_unknown_voice_is_a_bad_request_that_keeps_the_deployment_serving( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + respx_mock.post(_SYNTHESIZE_URL).mock( + side_effect=lambda request: ( + httpx.Response( + 404, json={"error": {"code": 404, "message": "Voice Koree not found.", "status": "NOT_FOUND"}} + ) + if json.loads(request.content)["voice"]["name"] == "Koree" + else httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "gemini-tts", + "litellm_params": { + "model": "vertex_ai/gemini-3.1-flash-tts-preview", + "vertex_credentials": _AUTHORIZED_USER, + "vertex_location": "global", + }, + } + ], + num_retries=0, + ) + + with pytest.raises(litellm.BadRequestError, match="Voice Koree not found"): + await router.aspeech(model="gemini-tts", input="Say hello", voice="Koree", response_format="mp3") + response: Final = await router.aspeech(model="gemini-tts", input="Say hello", voice="Kore", response_format="mp3") + + assert response.content == audio_bytes + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_chirp_unknown_voice_stays_not_found( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response( + 404, json={"error": {"code": 404, "message": "Voice en-US-Nope not found.", "status": "NOT_FOUND"}} + ) + ) + + with pytest.raises(litellm.NotFoundError, match="Voice en-US-Nope not found"): + await litellm.aspeech( + model="vertex_ai/chirp", + input="Say hello", + voice={"languageCode": "en-US", "name": "en-US-Nope"}, + vertex_credentials=_AUTHORIZED_USER, + ) + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_rejects_a_deployment_api_base( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + with pytest.raises(litellm.BadRequestError, match="Request wav or pcm"): + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Say hello", + voice="Kore", + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + vertex_location="us-central1", + api_base="https://vertex.gateway.internal", + ) + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_sends_speak_tags_as_text( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + speech_input: Final = "Have a wonderful day" + + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input=speech_input, + voice="Kore", + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + sent: Final = json.loads(route.calls[0].request.content) + assert sent["input"] == {"text": speech_input} + + +@pytest.mark.parametrize("response_format", ("aac", "flac", "pcm16", "WAV")) +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_unsupported_format_is_rejected_before_calling_google( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock, response_format: str +) -> None: + with pytest.raises(litellm.BadRequestError, match="Supported response formats: alaw, mp3, mulaw, opus, pcm, wav"): + await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Say hello", + voice="Kore", + response_format=response_format, + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_sends_both_speakers_to_cloud_tts( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + + result: Final = await litellm.aspeech( + model="vertex_ai/gemini-3.1-flash-tts-preview", + input="Ryan: Hi.\nKatie: Hello.", + voice={ + "multi_speaker_voice_config": { + "speaker_voice_configs": [ + {"speaker": "Ryan", "voice_config": {"prebuilt_voice_config": {"voice_name": "Umbriel"}}}, + {"speaker": "Katie", "voice_config": {"prebuilt_voice_config": {"voice_name": "Leda"}}}, + ], + }, + }, + response_format="mp3", + vertex_credentials=_AUTHORIZED_USER, + vertex_location="global", + ) + + assert result.content == audio_bytes + assert result.usage is not None + assert result.usage.prompt_tokens > 0 + assert result.usage.completion_tokens > 0 + assert route.call_count == 1 + sent: Final = json.loads(route.calls[0].request.content) + assert sent["audioConfig"]["audioEncoding"] == "MP3" + assert sent["voice"] == { + "languageCode": "en-US", + "modelName": "gemini-3.1-flash-tts-preview", + "multiSpeakerVoiceConfig": { + "speakerVoiceConfigs": [ + {"speakerAlias": "Ryan", "speakerId": "Umbriel"}, + {"speakerAlias": "Katie", "speakerId": "Leda"}, + ], + }, + } + + +@pytest.mark.parametrize( + ("speech_params", "error"), + ( + ({"voice": ["Kore"]}, "needs voice to be a Gemini voice name"), + ({"voice": ""}, "needs voice to be a Gemini voice name"), + ({"voice": {"name": "Kore", "style": "calm"}}, "does not support the voice keys style"), + ({"voice": {"speechConfig": {"voice": "Kore", "pitch": 2}}}, "does not support the voice keys pitch"), + ({"voice": "Kore", "instructions": ["Read slowly"]}, "needs instructions to be a string"), + ), +) +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_mp3_malformed_params_are_bad_requests_unless_dropped( + respx_mock: respx.MockRouter, + google_token_endpoint: responses.RequestsMock, + speech_params: dict[str, object], + error: str, +) -> None: + audio_bytes: Final = (Path(__file__).resolve().parents[4] / "audio_tests/gemini_tts_speech.mp3").read_bytes() + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(audio_bytes).decode()}) + ) + request: Final = { + "model": "vertex_ai/gemini-3.1-flash-tts-preview", + "input": "Say hello", + "response_format": "mp3", + "vertex_credentials": _AUTHORIZED_USER, + "vertex_location": "global", + **speech_params, + } + + with pytest.raises(litellm.BadRequestError, match=error): + await litellm.aspeech(**request) + assert not route.called + + response: Final = await litellm.aspeech(**request, drop_params=True) + + sent: Final = json.loads(route.calls[0].request.content) + assert response.content == audio_bytes + assert sent["input"] == {"text": "Say hello"} + assert sent["voice"] == {"languageCode": "en-US", "modelName": "gemini-3.1-flash-tts-preview", "name": "Kore"} + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_beta_lyria_2_serves_the_openai_params_it_ignores( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + route: Final = respx_mock.post(url__regex=r"https://us-central1-aiplatform\.googleapis\.com/.*/lyria-002:predict$").mock( + return_value=httpx.Response(200, json={"predictions": [{"audioContent": "bHlyaWEtMi1hdWRpbw==", "mimeType": "audio/wav"}]}) + ) + + response: Final = await litellm.aspeech( + model="vertex_ai_beta/lyria-002", + input="A bright synth track", + voice="alloy", + response_format="mp3", + speed=1.5, + vertex_credentials=_AUTHORIZED_USER, + vertex_project="test-project", + vertex_location="us-central1", + ) + + assert route.call_count == 1 + assert response.content == b"lyria-2-audio" + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_gemini_tts_wav_unknown_voice_keys_are_bad_requests_unless_dropped( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + route: Final = respx_mock.post( + url__regex=r"https://aiplatform\.googleapis\.com/.*/models/gemini-3.1-flash-tts-preview:generateContent$" + ).mock(return_value=httpx.Response(200, json=_GENERATE_CONTENT_RESPONSE)) + request: Final = { + "model": "vertex_ai/gemini-3.1-flash-tts-preview", + "input": "Say hello", + "voice": {"speech_config": {"voice": "Kore", "style": "calm"}, "modelName": "gemini-3.1-flash-tts-preview"}, + "response_format": "wav", + "vertex_credentials": _AUTHORIZED_USER, + "vertex_location": "global", + } + + with pytest.raises(litellm.BadRequestError, match="does not support the voice keys style"): + await litellm.aspeech(**request) + assert not route.called + + response: Final = await litellm.aspeech(**request, drop_params=True) + + assert _PCM_AUDIO in response.content + assert json.loads(route.calls[0].request.content)["generationConfig"]["speechConfig"] == { + "voiceConfig": {"prebuiltVoiceConfig": {"voiceName": "Kore"}} + } diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py index eb222366812..d47db69257b 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py @@ -722,6 +722,16 @@ def _router_with_executed_batch_model() -> Router: "litellm_params": {"model": "gemini/gemini-2.0-flash"}, "model_info": {"id": "gemini-2.0-flash-id"}, }, + { + "model_name": "vertex-batch-drop", + "litellm_params": {"model": "vertex_ai/gemini-2.5-flash", "drop_params": True}, + "model_info": {"id": "vertex-batch-drop-id"}, + }, + { + "model_name": "vertex-batch-keep", + "litellm_params": {"model": "vertex_ai/gemini-2.5-flash", "drop_params": False}, + "model_info": {"id": "vertex-batch-keep-id"}, + }, ] ) @@ -939,6 +949,39 @@ def test_batch_upload_for_a_provider_model_still_goes_to_the_provider(batch_uplo assert provider_upload.call_args.kwargs["custom_llm_provider"] == "gemini" +@pytest.mark.parametrize( + "model, expected_drop_params", + [("vertex-batch-drop", True), ("vertex-batch-keep", False)], +) +def test_batch_upload_for_a_provider_model_carries_the_deployment_drop_params( + batch_upload_seams, model: str, expected_drop_params: bool +): + _, provider_upload, _ = batch_upload_seams + + response = _upload_batch_file({"x-litellm-model": model}, {}) + + assert response.status_code == 200, response.text + assert provider_upload.call_args.kwargs["drop_params"] is expected_drop_params + + +def test_batch_upload_for_a_provider_model_without_deployment_drop_params_sends_no_drop_params(batch_upload_seams): + _, provider_upload, _ = batch_upload_seams + + response = _upload_batch_file({"x-litellm-model": "gemini-2.0-flash"}, {}) + + assert response.status_code == 200, response.text + assert "drop_params" not in provider_upload.call_args.kwargs + + +def test_batch_upload_keeps_the_callers_drop_params_over_the_deployment(batch_upload_seams): + _, provider_upload, _ = batch_upload_seams + + response = _upload_batch_file({"x-litellm-model": "vertex-batch-keep", "user-agent": "claude-cli/2.1.0"}, {}) + + assert response.status_code == 200, response.text + assert provider_upload.call_args.kwargs["drop_params"] is True + + @pytest.mark.skip(reason="mock respx fails on ci/cd - unclear why") def test_create_file_and_call_chat_completion_e2e( mocker: MockerFixture, monkeypatch, llm_router: Router diff --git a/tests/unit/test_cost_calculator.py b/tests/unit/test_cost_calculator.py index 2854f9010b2..5d4eb5fae12 100644 --- a/tests/unit/test_cost_calculator.py +++ b/tests/unit/test_cost_calculator.py @@ -31,7 +31,7 @@ from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.fireworks_ai.cost_calculator import get_base_model_for_pricing from litellm.llms.together_ai.cost_calculator import get_model_params_and_category from litellm.types.llms.base import CachedTokensDetails -from litellm.types.llms.openai import OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse +from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIRealtimeStreamList, ResponseAPIUsage, ResponsesAPIResponse from litellm.types.llms.xai import RealtimeMeteredUsage from litellm.types.rerank import RerankResponse from litellm.types.utils import CallTypes, Choices, CompletionTokensDetailsWrapper, EmbeddingResponse, ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails, LiteLLMRealtimeStreamLoggingObject, Message, ModelInfo, ModelResponse, PromptTokensDetails, PromptTokensDetailsWrapper, Usage @@ -9134,6 +9134,33 @@ def test_azure_tts_cost_uses_test_owned_character_rate(monkeypatch: pytest.Monke assert cost == pytest.approx(0.05) +def test_speech_cost_with_custom_pricing_bills_the_reported_usage() -> None: + usage: Final = Usage( + prompt_tokens=10, + completion_tokens=1000, + total_tokens=1010, + completion_tokens_details=CompletionTokensDetailsWrapper(audio_tokens=1000), + ) + response: Final = HttpxBinaryResponseContent( + httpx.Response(200, content=b"RIFF....WAVE", headers={"content-type": "audio/wav"}) + ) + response.usage = usage + rates: Final = MappingProxyType({"input_cost_per_token": 1e-6, "output_cost_per_token": 2e-5}) + + cost: Final = completion_cost( + completion_response=response, + model="vertex_ai/gemini-3.1-flash-tts-preview", + call_type="speech", + prompt="Say hello", + custom_cost_per_token=dict(rates), + ) + + assert cost == pytest.approx( + usage.prompt_tokens * rates["input_cost_per_token"] + + usage.completion_tokens * rates["output_cost_per_token"] + ) + + def test_moderation_cost_uses_test_owned_token_rate() -> None: prompt: Final = "moderation input" rates: Final = MappingProxyType( From 1fcbb0355e76065c5ed0e79aba298c8ece25033a Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:31:19 -0700 Subject: [PATCH 052/235] fix(pricing): update baseten reflection/Beam-1.0 prices from the models API (#47784) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++--- model_prices_and_context_window.json | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index fd72d209ffb..764cf397158 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -76709,14 +76709,14 @@ "supports_response_schema": true }, "baseten/reflection/Beam-1.0": { - "cache_read_input_token_cost": 1.3e-07, - "input_cost_per_token": 1.3e-06, + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 4e-07, "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 4.3e-06, + "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index fd72d209ffb..764cf397158 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -76709,14 +76709,14 @@ "supports_response_schema": true }, "baseten/reflection/Beam-1.0": { - "cache_read_input_token_cost": 1.3e-07, - "input_cost_per_token": 1.3e-06, + "cache_read_input_token_cost": 4e-08, + "input_cost_per_token": 4e-07, "litellm_provider": "baseten", "max_input_tokens": 1048576, "max_output_tokens": 131072, "max_tokens": 131072, "mode": "chat", - "output_cost_per_token": 4.3e-06, + "output_cost_per_token": 1.2e-06, "source": "https://inference.baseten.co/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, From 820725661e865983859d73bca9b9b712706c0371 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 04:46:45 +0000 Subject: [PATCH 053/235] fix(router): log each dropped /v1/messages attempt through its own logging copy and open the retry without its usage (#45194) * fix(proxy): keep a retried /v1/messages stream's spend log by logging the superseded attempt as a failure A /v1/messages stream the router retries before any content shares one logging object across both attempts. The superseded attempt's end-of-stream logging ran as a success (a usage-only or placeholder result) and claimed the request's one-shot success log, so the recovered stream's spend row was either missing or written under the superseded attempt's message id. The passthrough logging path now logs an Anthropic stream whose frames carry an `event: error` as a failure with whatever usage the provider reported, the way a dropped stream already does, and the router closes the superseded stream before it opens the retry so that logging is queued first. * fix(proxy): log a provider error frame through an attempt-scoped logging copy The error-frame failure stashed the superseded attempt's usage on the logging object the retry shares, so the retry's success log carried the failed attempt's token counts. The failure now logs through a copy of the logging object with its own model_call_details, and the helper moves into the Anthropic passthrough logging handler so the proxy streaming handler only routes by endpoint type. * fix(proxy): scope the error-frame failure log to streams closed at the frame A stream that ran to its end at the provider's error frame kept logging through the success route before this PR, and the attempt-scoped failure log is now taken only for a stream the router closed at that frame, with the copy made at the close so the retry's deployment never reaches it * fix(router): log each dropped /v1/messages attempt through its own logging copy and open the retry without its usage A /v1/messages stream dropped after message_start and retried in its model group shares one Logging object across attempts, so the retry's success log inherited the dropped attempt's stashed usage and exception, and a second drop never reached the failure handlers because the first had already marked the object as logged. The dropped attempt now dispatches its failure handlers through an attempt-scoped copy, and the router discards the stashed usage right before it opens the next attempt, on same-group retries and on every fallback hop, so a request that runs out of attempts still bills the last attempt's partial usage. * fix(router): dispatch chat stream failures through the attempt-scoped logging copy The router now clears a dropped attempt's stashed usage before every fallback hop, so the chat stream wrapper's queued failure log must read that usage from its own copy of the logging object, like the /v1/messages path already does. The stash test asserts through a real success log, the cooldown test reads a real Router's cooldown list, and the integration helpers carry full types. * test(vertex): give the stream wrapper's logging stub an attempt_scoped_copy * test: route the pass-through logging mock's attempt copy to itself and read the callback sink through a reader The pass-through failure test stubbed dispatch_failure_handlers on a spec'd MagicMock, which the attempt-scoped copy no longer reached, so the logging worker never finished. The integration tests kept earlier sink batches in a caller-owned list the helper mutated; a small reader keeps them instead. * test(router): use real Logging in fallback cooldown test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): preserve superseded retry usage for billing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): bill retry input after prior stream usage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): track retry dispatch for terminal billing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): use deterministic retry timestamp Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): avoid real clock in fallback tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(tests): remove duplicate streaming logger import Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): retain error-frame usage for exhausted retries * test(proxy): fix retry billing expectations and helper annotations * fix(proxy): preserve failure estimate without carried usage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): audit retry helpers and fixtures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): fix retry fixture timestamps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): observe retry stash at attempt boundary Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): keep terminal failure attribution when dropped attempts log on copies Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): publish a dropped attempt's payload only while it is still the current attempt Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): cover spend-log encoding error propagation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): skip adapter failure dispatch when the wrapped stream already logged it Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): cool down the final fallback attempt when its failure callbacks were blocked Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): import Final in adapter regression Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): drop spend-log encoding expectation superseded by main's poison isolation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): finish the wrapped stream's failure log before surfacing the error Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): wait for the wrapped stream's failure log to finish like the inline dispatch did Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(anthropic): wait for the executor-run sync failure callback separately from the async caller boundary Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): pin pass-through stream fixture start times Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): publish failed attempt payload before awaiting failure callbacks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(logging): publish failed attempt payload from the failure handler after baseline invalidation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(logging): drop unused discard_partial_usage_for_failure Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/litellm_logging.py | 46 +- .../litellm_core_utils/streaming_handler.py | 33 +- .../adapters/streaming_iterator.py | 26 +- .../anthropic_passthrough_logging_handler.py | 22 - .../streaming_handler.py | 50 +- litellm/proxy/utils.py | 30 +- litellm/router.py | 2 + .../router_utils/fallback_event_handlers.py | 31 +- ...thropic_messages_pre_content_retry_wire.py | 163 +++- .../test_litellm_logging.py | 6 + .../test_max_streaming_duration.py | 11 +- .../test_streaming_handler.py | 66 +- ...est_streaming_iterator_mid_stream_error.py | 81 +- ...test_vertex_and_google_ai_studio_gemini.py | 8 + ...t_anthropic_passthrough_logging_handler.py | 51 -- .../test_streaming_handler.py | 4 + .../test_streaming_handler_interrupt.py | 145 +++- ..._utils_model_creation_and_error_logging.py | 726 +++++++++++++++++- .../mcp/test_aresponses_api_with_mcp.py | 19 +- .../test_fallback_event_handlers.py | 233 +++++- tests/unit/test_router/test_router.py | 52 +- 21 files changed, 1631 insertions(+), 174 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a59672b5cb2..d75d35e4531 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -604,6 +604,10 @@ def _timestamp_seconds(moment: object) -> float | None: return None +def same_attempt_marker(first: object, second: object) -> bool: + return first is second or first == second + + class Logging(LiteLLMLoggingBaseClass): global \ supabaseClient, \ @@ -625,6 +629,7 @@ class Logging(LiteLLMLoggingBaseClass): classifier_input: Mapping[str, JsonValue] | None = None baseline_cache_context: "BaselineCacheContext | None" = None baseline_observation: "CapturedBaselineObservation | None" = None + _on_failure_payload_built: "Callable[[], None] | None" = None @property def _defer_async_logging(self) -> bool: @@ -852,6 +857,7 @@ class Logging(LiteLLMLoggingBaseClass): "litellm_call_id": litellm_call_id, "start_time": start_time, "first_api_call_start_time": None, + "current_attempt_api_call_start_time": None, "litellm_params": litellm_params, } new_call.streaming_chunks = [] @@ -1602,7 +1608,9 @@ class Logging(LiteLLMLoggingBaseClass): # A proxy-gate failure (auth, rate limit, client disconnect) fires pre_call for its callbacks # without any provider handoff, so it must not stamp the handoff instants. if LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL not in self.model_call_details: - self.model_call_details["api_call_start_time"] = datetime.datetime.now() + api_call_start_time: Final = datetime.datetime.now() + self.model_call_details["api_call_start_time"] = api_call_start_time + self.model_call_details["current_attempt_api_call_start_time"] = api_call_start_time # Set-once first provider-handoff instant. api_call_start_time # is overwritten on every retry, so it can't measure one-time # preprocessing; pinning the first attempt excludes retry loops @@ -1610,9 +1618,7 @@ class Logging(LiteLLMLoggingBaseClass): # litellm_params["metadata"] (caller request metadata, typed # Dict[str, str], echoed downstream; a datetime breaks it). if self.model_call_details.get("first_api_call_start_time") is None: - self.model_call_details["first_api_call_start_time"] = self.model_call_details[ - "api_call_start_time" - ] + self.model_call_details["first_api_call_start_time"] = api_call_start_time # Input Integration Logging -> If you want to log the fact that an attempt to call the model was made callbacks: Final = ( *self._without_replaced_global_callbacks(litellm.input_callback), @@ -2474,12 +2480,37 @@ class Logging(LiteLLMLoggingBaseClass): attempt.model_call_details = dict(self.model_call_details) return attempt + def dispatch_attempt_failure_handlers( + self, exception: Exception, traceback_exception: str + ) -> Coroutine[None, None, None]: + shared_payload: Final = self.model_call_details.get("standard_logging_object") + attempt_marker: Final = self.model_call_details.get("current_attempt_api_call_start_time") + attempt: Final = self.attempt_scoped_copy() + + def publish() -> None: + attempt_payload: Final = attempt.model_call_details.get("standard_logging_object") + current_marker: Final = self.model_call_details.get("current_attempt_api_call_start_time") + if ( + attempt_payload is not None + and self.model_call_details.get("standard_logging_object") is shared_payload + and same_attempt_marker(current_marker, attempt_marker) + ): + self.model_call_details["standard_logging_object"] = attempt_payload + + attempt._on_failure_payload_built = publish + + async def dispatch() -> None: + await attempt.dispatch_failure_handlers(exception, traceback_exception, prefer_async_handlers=True) + + return dispatch() + def carry_failed_attempt_usage(self) -> None: """Move what a failed earlier attempt stashed off the retry sharing this logging object, so the retry logs its own usage while the request's failure log still bills every attempt.""" usage: Final = self.model_call_details.pop("combined_usage_object", None) response_cost: Final = self.model_call_details.pop("response_cost", None) if isinstance(usage, Usage): + self.model_call_details["current_attempt_api_call_start_time"] = None self.model_call_details["failed_attempts_usage"] = ( *self.model_call_details.get("failed_attempts_usage", ()), (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0), @@ -2593,6 +2624,10 @@ class Logging(LiteLLMLoggingBaseClass): """ return self.model_call_details[f"has_logged_{event_type}"] = True + if event_type == "async_failure": + self.model_call_details["async_failure_logged_attempt"] = self.model_call_details.get( + "current_attempt_api_call_start_time" + ) return def has_run_logging( @@ -4410,7 +4445,8 @@ class Logging(LiteLLMLoggingBaseClass): start_time=start_time, end_time=end_time, ) - + if self._on_failure_payload_built is not None: + self._on_failure_payload_built() callbacks: Final = self.get_combined_callback_list( dynamic_success_callbacks=self.dynamic_async_failure_callbacks, global_callbacks=litellm._async_failure_callback, diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 5e43157a55d..f994baa52b1 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -2283,12 +2283,7 @@ class CustomStreamWrapper: traceback_exception = traceback.format_exc() ## ADD DEBUG INFORMATION - E.G. LITELLM REQUEST TIMEOUT traceback_exception += f"\nLiteLLM Default Request Timeout - {litellm.request_timeout}" - if self.logging_obj is not None: - self._record_partial_usage_for_failure() - ## LOGGING - self._failure_logging = asyncio.create_task( - self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True) - ) + self._dispatch_stream_failure(e, traceback_exception) await self.aclose() self._handle_stream_fallback_error(e) except (httpx.ReadError, httpx.RemoteProtocolError) as e: @@ -2389,16 +2384,20 @@ class CustomStreamWrapper: return processed_chunk def _log_stream_failure_and_raise(self, e: Exception) -> NoReturn: - traceback_exception: Final = traceback.format_exc() - if self.logging_obj is not None: - self._record_partial_usage_for_failure() - ## LOGGING - self._failure_logging = asyncio.create_task( - self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True) - ) + self._dispatch_stream_failure(e, traceback.format_exc()) self._handle_stream_fallback_error(e) - async def wait_for_failure_logging(self, timeout: float) -> None: + def _dispatch_stream_failure(self, e: Exception, traceback_exception: str) -> None: + self._record_partial_usage_for_failure() + self._failure_logging = asyncio.create_task( + self.logging_obj.dispatch_attempt_failure_handlers(e, traceback_exception) + ) + + @property + def has_dispatched_failure_logging(self) -> bool: + return self._failure_logging is not None + + async def wait_for_failure_logging(self, timeout: float | None) -> None: """Let this stream's failure log read its own deployment before a fallback reuses the logging object.""" if self._failure_logging is None: return @@ -2412,9 +2411,9 @@ class CustomStreamWrapper: A stream that breaks mid-flight still billed the provider for the chunks already delivered. Recover that partial usage from the chunks seen so far and stash it, with its cost, on the logging object so the failure - handler records the real partial spend instead of zero. A request that - later recovers via a router fallback overwrites this with the combined - success log on the same request id, so this never double counts. + handler records the real partial spend instead of zero. The router + moves it into terminal failure usage before opening the next attempt, + so a recovered attempt logs its own usage. """ if self.logging_obj is None or not self.chunks: return diff --git a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py index 52b836c789a..e9fd77ef441 100644 --- a/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py +++ b/litellm/llms/anthropic/pass_through/adapters/streaming_iterator.py @@ -16,6 +16,7 @@ from typing import ( Protocol, cast, get_args, + runtime_checkable, ) from pydantic import TypeAdapter @@ -65,6 +66,14 @@ class _ChunkStream(Protocol): def __aiter__(self) -> "AsyncIterator[ModelResponseStream]": ... +@runtime_checkable +class _FailureLoggingWaiter(Protocol): + @property + def has_dispatched_failure_logging(self) -> bool: ... + + async def wait_for_failure_logging(self, timeout: float | None) -> None: ... + + def _optional_attr(obj: object, name: str) -> object: return getattr(obj, name, None) @@ -189,6 +198,14 @@ class _CombinedChunkSplitter: self._held_calls: Mapping[int, tuple[ModelResponseStream, _ToolCallPart]] = MappingProxyType({}) self._live_call_index: int | None = None + @property + def has_dispatched_failure_logging(self) -> bool: + return isinstance(self._stream, _FailureLoggingWaiter) and self._stream.has_dispatched_failure_logging is True + + async def wait_for_failure_logging(self, timeout: float | None) -> None: + if isinstance(self._stream, _FailureLoggingWaiter): + await self._stream.wait_for_failure_logging(timeout=timeout) + @property def chunks(self) -> "list[ModelResponseStream] | None": return cast( # cast-ok: chunks is a list of ModelResponseStream on the inner stream @@ -522,7 +539,9 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): # Wrap the upstream stream so chunks that carry both content and a # finish_reason (fake-streamed providers) are split into two — see # _CombinedChunkSplitter. - super().__init__(_CombinedChunkSplitter(completion_stream)) + combined_chunk_splitter: Final = _CombinedChunkSplitter(completion_stream) + super().__init__(combined_chunk_splitter) + self._combined_chunk_splitter: Final = combined_chunk_splitter self.model = model self._message_id: str = f"msg_{uuid.uuid4()}" if litellm_logging_obj is not None: @@ -1244,7 +1263,10 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper): if provider_error is e: raise raise provider_error from e - if logging_obj is not None: + stream_has_dispatched_failure_logging: Final = self._combined_chunk_splitter.has_dispatched_failure_logging + if stream_has_dispatched_failure_logging: + await self._combined_chunk_splitter.wait_for_failure_logging(timeout=None) + elif logging_obj is not None: try: await logging_obj.dispatch_failure_handlers( exception=provider_error, diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index a7f3a4f84a7..23ef89b3c5c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -10,14 +10,12 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import ANTHROPIC_BATCHES_ROUTE -from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_content_from_model_response, ) from litellm.llms.anthropic import get_anthropic_config -from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception from litellm.llms.anthropic.pass_through.stream_assembly import ( build_complete_streaming_response, build_complete_streaming_response_legacy, @@ -256,26 +254,6 @@ class AnthropicPassthroughLoggingHandler: ), ) - @staticmethod - async def log_error_frame_as_failure( - attempt_logging_obj: LiteLLMLoggingObj, - request_body: Mapping[str, object], - all_chunks: Sequence[str | bytes], - error_event: tuple[str, str, int], - ) -> None: - """A stream closed at the provider's `event: error` frame is one failed attempt of the request: it logs - through the attempt-scoped copy its caller took at the close, so the usage it consumed never reaches - the success log of the attempt the router opens in its place.""" - error_type, message, status_code = error_event - await asyncify(AnthropicPassthroughLoggingHandler.record_partial_usage_for_failure)( - litellm_logging_obj=attempt_logging_obj, request_body=request_body, all_chunks=all_chunks - ) - await attempt_logging_obj.dispatch_failure_handlers( - anthropic_error_frame_exception(error_type, message, status_code, str(attempt_logging_obj.model)), - "", - prefer_async_handlers=True, - ) - @staticmethod def _cost_partial_stream_or_zero( partial_response: ModelResponse | TextCompletionResponse, model: str, logging_obj: LiteLLMLoggingObj diff --git a/litellm/proxy/pass_through_endpoints/streaming_handler.py b/litellm/proxy/pass_through_endpoints/streaming_handler.py index 1fcfe2e279a..b6a067b7666 100644 --- a/litellm/proxy/pass_through_endpoints/streaming_handler.py +++ b/litellm/proxy/pass_through_endpoints/streaming_handler.py @@ -1,5 +1,5 @@ import traceback -from collections.abc import Coroutine, Sequence +from collections.abc import Callable, Coroutine, Mapping, Sequence from dataclasses import dataclass from datetime import datetime, timezone from typing import Final, Protocol @@ -12,6 +12,7 @@ from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import bind_budget_reservation_to_callbacks from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.llms.anthropic.common_utils import anthropic_error_frame_exception from litellm.proxy._types import PassThroughEndpointLoggingResultValues from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.common_utils.sse_keepalive import split_complete_sse_frames @@ -87,13 +88,32 @@ class PassThroughStreamingHandler: ) try: GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - async_coroutine=litellm_logging_obj.dispatch_failure_handlers( - exception, traceback.format_exc(), prefer_async_handlers=True - ) + async_coroutine=litellm_logging_obj.dispatch_attempt_failure_handlers(exception, traceback.format_exc()) ) except Exception as e: verbose_proxy_logger.error("Error scheduling stream failure logging: %s", e) + @staticmethod + async def _schedule_stream_end_logging( + litellm_logging_obj: LiteLLMLoggingObj, + endpoint_type: EndpointType, + request_body: Mapping[str, object], + raw_bytes: Sequence[bytes], + error_event: tuple[str, str, int] | None, + success_logging: Callable[[], Coroutine[None, None, None]], + ) -> None: + if error_event is None: + GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(async_coroutine=success_logging()) + return + error_type, message, status_code = error_event + await PassThroughStreamingHandler.schedule_stream_failure_logging( + litellm_logging_obj=litellm_logging_obj, + endpoint_type=endpoint_type, + request_body=dict(request_body), + raw_bytes=raw_bytes, + exception=anthropic_error_frame_exception(error_type, message, status_code, str(litellm_logging_obj.model)), + ) + @staticmethod def _record_partial_usage_for_failure( litellm_logging_obj: LiteLLMLoggingObj, @@ -276,17 +296,13 @@ class PassThroughStreamingHandler: else PassThroughStreamingHandler._error_frame_closed_at(endpoint_type, raw_bytes) ) try: - GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue( - async_coroutine=( - _build_logging_coroutine() - if superseded_error_event is None - else AnthropicPassthroughLoggingHandler.log_error_frame_as_failure( - attempt_logging_obj=litellm_logging_obj.attempt_scoped_copy(), - request_body=resolved_request_body, - all_chunks=tuple(raw_bytes), - error_event=superseded_error_event, - ) - ) + await PassThroughStreamingHandler._schedule_stream_end_logging( + litellm_logging_obj=litellm_logging_obj, + endpoint_type=endpoint_type, + request_body=resolved_request_body, + raw_bytes=tuple(raw_bytes), + error_event=superseded_error_event, + success_logging=_build_logging_coroutine, ) except Exception as e: verbose_proxy_logger.error("Error scheduling chunk_processor logging: %s", e) @@ -295,10 +311,6 @@ class PassThroughStreamingHandler: @staticmethod def _error_frame_closed_at(endpoint_type: EndpointType, raw_bytes: Sequence[bytes]) -> tuple[str, str, int] | None: - """The provider `event: error` frame of a stream the router closed because a retry or fallback took the - request over. That attempt is logged as its own failure, on a copy taken now so the retry's rebinding of - the shared logging object never reaches it. A stream that ran to its end or that the client closed keeps - logging through the success route, so its usage is still billed.""" from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( closing_superseded_attempt, parse_anthropic_error_event, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 6c9b23860da..2a328971817 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1015,7 +1015,7 @@ def _billed_across_attempts( ) -def _failure_usage_to_lift( +def _current_failure_usage_to_lift( model_call_details: Mapping[str, object], request_body: Mapping[str, object], dispatched: bool, @@ -1037,9 +1037,6 @@ def _failure_usage_to_lift( failure stays at zero cost. Returns the (combined_usage_object, response_cost) pair to lift, or None.""" recovered_usage: Final = model_call_details.get("combined_usage_object") - failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) - if failed_attempts: - return _billed_across_attempts(failed_attempts, recovered_usage, model_call_details.get("response_cost")) if recovered_usage is not None: return recovered_usage, model_call_details.get("response_cost") if not dispatched: @@ -1281,6 +1278,29 @@ def _estimated_input_cost( return next((cost for cost in candidate_costs if cost is not None), 0.0) +def _failure_usage_to_lift( + model_call_details: Mapping[str, object], + request_body: Mapping[str, object], + dispatched: bool, + provider_billed: bool = False, + deployment_id: str | None = None, + base_model: str | None = None, +) -> tuple[object, object] | None: + current_attempt: Final = _current_failure_usage_to_lift( + model_call_details, + request_body, + dispatched, + provider_billed, + deployment_id, + base_model, + ) + failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) + if not failed_attempts: + return current_attempt + current_usage, current_cost = current_attempt if current_attempt is not None else (None, None) + return _billed_across_attempts(failed_attempts, current_usage, current_cost) + + _EMPTY_LIFT: Final = MappingProxyType({}) @@ -1444,7 +1464,7 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: _usage_to_lift: Final = _failure_usage_to_lift( model_call_details=_model_call_details, request_body=request_data, - dispatched=_first_handoff is not None, + dispatched=_model_call_details.get("current_attempt_api_call_start_time", _first_handoff) is not None, provider_billed=_billed, deployment_id=_deployment_id, base_model=_base_model, diff --git a/litellm/router.py b/litellm/router.py index 0ac21960be1..fe418b5830b 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -260,6 +260,7 @@ from litellm.router_utils.fallback_event_handlers import ( attempted_retries_for_request, carry_over_pre_routing_selection, carry_over_routed_deployment, + carry_superseded_attempt_usage, check_non_standard_fallback_format, clear_pre_routing_selection, committed_retry_budget_for_request, @@ -6721,6 +6722,7 @@ class Router: verbose_router_logger.debug( "Retrying stream that failed before content, attempt %s of %s", attempt + 1, budget ) + carry_superseded_attempt_usage(retry_kwargs) try: response = await attempt_function(**retry_kwargs) except Exception as retry_error: # noqa: BLE001 # the fallback chain judges the failed retry diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index e1201df1a13..6c93fe6d239 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -8,6 +8,7 @@ from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final import httpx +from pydantic import TypeAdapter import litellm from litellm._logging import verbose_router_logger @@ -15,6 +16,7 @@ from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_metadata_variable_name_from_kwargs, safe_deep_copy from litellm.litellm_core_utils.credential_accessor import reject_api_key_secret_reference from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider +from litellm.litellm_core_utils.litellm_logging import Logging, same_attempt_marker from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_structure from litellm.router_utils.add_retry_fallback_headers import ( add_fallback_headers_to_response, @@ -51,6 +53,7 @@ else: # (e.g. a nonexistent file/batch/thread id), independent of the selected deployment's health. _REQUEST_SCOPED_STATUS_CODES: Final = frozenset((404,)) _NO_MODEL_CALL_DETAILS: Final[Mapping[str, object]] = MappingProxyType({}) +_FALLBACK_KWARGS_ADAPTER: Final = TypeAdapter(dict[str, object]) def _trigger_cooldown_for_failed_deployment( @@ -447,6 +450,24 @@ def mid_stream_retry_kwargs( return {key: value for key, value in hop_kwargs.items() if key not in _MID_STREAM_RETRY_STRIPPED_KEYS} +def carry_superseded_attempt_usage(kwargs: Mapping[str, object]) -> None: + """Carry the superseded stream's partial usage into billing if the request ultimately fails.""" + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, Logging): + logging_obj.carry_failed_attempt_usage() + + +def _failure_callbacks_blocked_logging_object(kwargs: Mapping[str, object]) -> Logging | None: + logging_obj: Final = kwargs.get("litellm_logging_obj") + if not isinstance(logging_obj, Logging): + return None + if logging_obj.model_call_details.get("has_logged_async_failure", False) is not True: + return None + current_marker: Final = logging_obj.model_call_details.get("current_attempt_api_call_start_time") + logged_marker: Final = logging_obj.model_call_details.get("async_failure_logged_attempt") + return logging_obj if not same_attempt_marker(current_marker, logged_marker) else None + + def _request_metadata_bucket(kwargs: Mapping[str, object]) -> Mapping[str, object] | None: bucket: Final = kwargs.get(get_metadata_variable_name_from_kwargs(kwargs)) return bucket if isinstance(bucket, Mapping) else None @@ -869,6 +890,7 @@ async def run_async_fallback( kwargs["attempted_targets"] = attempted if include_fallback_errors: kwargs["include_fallback_errors"] = include_fallback_errors + carry_superseded_attempt_usage(_FALLBACK_KWARGS_ADAPTER.validate_python(kwargs)) response = await litellm_router.async_function_with_fallbacks(*args, **kwargs) verbose_router_logger.info("Successful fallback b/w models.") response = add_fallback_headers_to_response( @@ -891,13 +913,16 @@ async def run_async_fallback( kwargs=kwargs, original_exception=original_exception, ) - logging_obj = kwargs.get("litellm_logging_obj") - if logging_obj is not None and logging_obj.model_call_details.get("has_logged_async_failure", False): + if ( + failure_callbacks_blocked_logging_obj := _failure_callbacks_blocked_logging_object( + _FALLBACK_KWARGS_ADAPTER.validate_python(kwargs) + ) + ) is not None: _trigger_cooldown_for_failed_deployment( litellm_router=litellm_router, kwargs=kwargs, exception=e, - model_call_details=logging_obj.model_call_details, + model_call_details=failure_callbacks_blocked_logging_obj.model_call_details, ) raise error_from_fallbacks diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py index e0b2e4a336f..041a0a72fdd 100644 --- a/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py +++ b/tests/integration/messages_endpoint/providers/anthropic/test_anthropic_messages_pre_content_retry_wire.py @@ -1,13 +1,17 @@ import json import os import uuid -from collections.abc import Callable, Mapping +from collections.abc import Callable, Iterator, Mapping, Sequence +from contextlib import AbstractContextManager from dataclasses import dataclass +from pathlib import Path from typing import Final, Literal import anthropic import httpx +from pydantic import JsonValue, TypeAdapter import pytest +import yaml from integration._support.anthropic_sse import ( LIFECYCLE, Attempts, @@ -27,13 +31,14 @@ from integration._support.anthropic_sse import ( ) from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows +from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server -from pydantic import JsonValue from redis import Redis _MODEL: Final = "claude-sonnet-4-5-20250929" _API_KEY: Final = "synthetic-anthropic-key" _TEXT: Final = "Hello" +_SINK_EVENT_BATCH_ADAPTER: Final = TypeAdapter(list[dict[str, JsonValue]]) FirstAttempt = Literal[ "drop_after_headers", @@ -127,24 +132,24 @@ def _stream(gateway: Gateway, body: Mapping[str, JsonValue]) -> tuple[int, tuple return response.status_code, parse_sse(response.text) -def _rows(request_id: str) -> list[dict[str, JsonValue]]: - return read_rows('SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,)) +def _rows(gateway: Gateway, request_id: str) -> list[dict[str, JsonValue]]: + return gateway.rows('SELECT status, model_group FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (request_id,)) -def _success_rows(request_id: str) -> list[dict[str, JsonValue]]: - return eventually(lambda: _rows(request_id), lambda rows: len(rows) >= 1, seconds=70) +def _success_rows(gateway: Gateway, request_id: str) -> list[dict[str, JsonValue]]: + return eventually(lambda: _rows(gateway, request_id), lambda rows: len(rows) >= 1, seconds=70) def _assert_completed_by_retry( - status: int, events: tuple[SseEvent, ...], prompt: str, model: str, wire: Wire, *, attempts: int = 2 + gateway: Gateway, status: int, events: tuple[SseEvent, ...], prompt: str, model: str, wire: Wire, *, attempts: int = 2 ) -> None: assert status == 200, events assert event_types(events) == LIFECYCLE, events assert message_id(events) == _served_id(prompt, attempts), events assert delta_text(events) == _TEXT, events assert [request.target for request in wire.drain()] == ["/v1/messages"] * attempts - assert _success_rows(_served_id(prompt, attempts)) == [{"status": "success", "model_group": model}] - superseded_rows: Final = {attempt: _rows(_served_id(prompt, attempt)) for attempt in range(1, attempts)} + assert _success_rows(gateway, _served_id(prompt, attempts)) == [{"status": "success", "model_group": model}] + superseded_rows: Final = {attempt: _rows(gateway, _served_id(prompt, attempt)) for attempt in range(1, attempts)} assert not any(superseded_rows.values()), superseded_rows @@ -183,7 +188,7 @@ def test_stream_failing_before_content_is_retried_on_the_same_group_and_complete **({"num_retries": 1} if source == "deployment" else {}), ) status, events = _stream(gateway, _body(model, prompt, source)) - _assert_completed_by_retry(status, events, prompt, model, wire) + _assert_completed_by_retry(gateway, status, events, prompt, model, wire) @pytest.mark.parametrize("source", ["deployment", "request"]) @@ -204,7 +209,7 @@ def test_stream_rejected_before_it_opens_is_retried_and_completes( **({"num_retries": 1} if source == "deployment" else {}), ) status, events = _stream(gateway, _body(model, prompt, source)) - _assert_completed_by_retry(status, events, prompt, model, wire) + _assert_completed_by_retry(gateway, status, events, prompt, model, wire) def _sdk(gateway: Gateway) -> anthropic.Anthropic: @@ -232,7 +237,7 @@ def test_anthropic_sdk_sync_stream_completes_after_a_drop_following_message_star assert final.id == _served_id(prompt, 2), final assert [block.text for block in final.content if block.type == "text"] == [_TEXT], final assert [request.target for request in wire.drain()] == ["/v1/messages"] * 2 - assert _success_rows(final.id) == [{"status": "success", "model_group": model}] + assert _success_rows(gateway, final.id) == [{"status": "success", "model_group": model}] async def test_anthropic_sdk_async_stream_completes_after_a_drop_following_message_start(gateway: Gateway) -> None: @@ -321,7 +326,7 @@ def test_stream_error_frame_after_content_is_not_retried_and_keeps_its_spend_row assert message_id(events) == _served_id(prompt, 1), events assert delta_text(events) == _TEXT, events assert [request.target for request in wire.drain()] == ["/v1/messages"] - assert [row["model_group"] for row in _success_rows(_served_id(prompt, 1))] == [model] + assert [row["model_group"] for row in _success_rows(gateway, _served_id(prompt, 1))] == [model] def test_stream_rejected_with_401_before_it_opens_is_not_retried(gateway: Gateway) -> None: @@ -393,6 +398,22 @@ def test_retry_rejected_before_it_opens_counts_against_the_same_budget(gateway: model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1) response: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "deployment")) _assert_rejected_before_content(response, wire, status=500, attempts=2, error="error") + rows: Final = eventually( + lambda: _request_rows(gateway, response.headers["x-litellm-call-id"]), + lambda values: len(values) >= 1, + seconds=70, + ) + rejected: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "request", budget=0)) + _assert_rejected_before_content(rejected, wire, status=500, attempts=1, error="error") + rejection_rows: Final = eventually( + lambda: _request_rows(gateway, rejected.headers["x-litellm-call-id"]), + lambda values: len(values) >= 1, + seconds=70, + ) + estimated_input: Final = rejection_rows[0]["prompt_tokens"] + assert isinstance(estimated_input, int) + assert rejection_rows == [{"status": "failure", "prompt_tokens": estimated_input, "completion_tokens": 0}] + assert rows == [{"status": "failure", "prompt_tokens": 5 + estimated_input, "completion_tokens": 1}], rows def test_non_stream_messages_rejected_with_500_is_retried_per_the_deployment_budget(gateway: Gateway) -> None: @@ -409,7 +430,7 @@ def test_non_stream_messages_rejected_with_500_is_retried_per_the_deployment_bud assert payload["id"] == _served_id(prompt, 2), response.text assert payload["content"] == [{"type": "text", "text": _TEXT}], response.text assert [request.target for request in wire.drain()] == ["/v1/messages"] * 2 - assert _success_rows(_served_id(prompt, 2)) == [{"status": "success", "model_group": model}] + assert _success_rows(gateway, _served_id(prompt, 2)) == [{"status": "success", "model_group": model}] def test_retried_stream_response_headers_name_the_attempt(gateway: Gateway) -> None: @@ -422,7 +443,7 @@ def test_retried_stream_response_headers_name_the_attempt(gateway: Gateway) -> N model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1) response: Final = gateway.request("POST", "/v1/messages", _body(model, prompt, "deployment")) events: Final = parse_sse(response.text) - _assert_completed_by_retry(response.status_code, events, prompt, model, wire) + _assert_completed_by_retry(gateway, response.status_code, events, prompt, model, wire) assert response.headers.get("x-litellm-attempted-retries") == "1", dict(response.headers) assert response.headers.get("x-litellm-max-retries") == "1", dict(response.headers) @@ -454,7 +475,7 @@ def test_retried_stream_spend_row_records_the_attempt_count(gateway: Gateway) -> ): model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1) status, events = _stream(gateway, _body(model, prompt, "deployment")) - _assert_completed_by_retry(status, events, prompt, model, wire) + _assert_completed_by_retry(gateway, status, events, prompt, model, wire) rows: Final = read_rows( "SELECT metadata->>'attempted_retries' AS attempted, metadata->>'max_retries' AS budget " 'FROM "LiteLLM_SpendLogs" WHERE request_id=%s', @@ -463,6 +484,114 @@ def test_retried_stream_spend_row_records_the_attempt_count(gateway: Gateway) -> assert rows == [{"attempted": "1", "budget": "1"}], rows +def _proxy_with_callback_sink(gateway: Gateway, tmp_path: Path, sink: Wire) -> AbstractContextManager[Gateway]: + loaded_config: Final = TypeAdapter(dict[str, object]).validate_python( + yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + ) + litellm_settings: Final = TypeAdapter(dict[str, object]).validate_python(loaded_config["litellm_settings"]) + config: Final = { + **loaded_config, + "litellm_settings": { + **litellm_settings, + "callbacks": ["generic_api"], + "DEFAULT_FLUSH_INTERVAL_SECONDS": 1, + }, + } + path: Final = tmp_path / "callbacks.yaml" + path.write_text(yaml.safe_dump(config)) + return owned_proxy( + gateway, + tmp_path, + {"GENERIC_LOGGER_ENDPOINT": sink.url, "GENERIC_LOGGER_HEADERS": "Authorization=Bearer synthetic-sink-secret"}, + config=path, + ) + + +def _sink_events(batches: Sequence[Request]) -> Iterator[dict[str, JsonValue]]: + for batch in batches: + yield from _SINK_EVENT_BATCH_ADAPTER.validate_python(json.loads(batch.body)) + + +class _SinkReader: + """Reads the callback sink across polls: drain() consumes the sink's queue, so earlier batches are kept here.""" + + def __init__(self, sink: Wire, model: str) -> None: + self._sink: Final = sink + self._model: Final = model + self._batches: tuple[Request, ...] = () + + def delivered_usage(self) -> tuple[tuple[str, int, int], ...]: + self._batches = (*self._batches, *self._sink.drain()) + return tuple( + (str(event["status"]), int(event["prompt_tokens"]), int(event["completion_tokens"])) + for event in _sink_events(self._batches) + if event.get("model_group") == self._model + ) + + +def _request_rows(gateway: Gateway, call_id: str) -> list[dict[str, JsonValue]]: + return gateway.rows( + 'SELECT status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s', (call_id,) + ) + + +@pytest.mark.parametrize( + "failure", + (_Failure("drop_after_message_start"), _Failure("error_frame_after_message_start", 500)), + ids=("transport-drop", "recoverable-error-frame"), +) +def test_superseded_attempt_and_its_retry_each_log_their_own_usage( + gateway: Gateway, tmp_path: Path, failure: _Failure +) -> None: + prompt: Final = _prompt() + attempts: Final = Attempts() + with ( + wire_server(_upstream(prompt, failure, attempts)) as wire, + wire_server(lambda request: Reply()) as sink, + _proxy_with_callback_sink(gateway, tmp_path, sink) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1) + status, events = _stream(candidate, _body(model, prompt, "deployment")) + _assert_completed_by_retry(candidate, status, events, prompt, model, wire) + delivered: Final = eventually( + _SinkReader(sink, model).delivered_usage, lambda values: len(values) >= 2, seconds=20 + ) + assert sorted(delivered) == [("failure", 5, 1), ("success", 5, 3)], delivered + + +def test_every_dropped_attempt_logs_its_own_failure_and_the_request_row_still_bills_the_dropped_usage( + gateway: Gateway, tmp_path: Path +) -> None: + prompt: Final = _prompt() + attempts: Final = Attempts() + with ( + wire_server(_upstream(prompt, _Failure("drop_after_message_start"), attempts, failing_attempts=99)) as wire, + wire_server(lambda request: Reply()) as sink, + _proxy_with_callback_sink(gateway, tmp_path, sink) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1) + response: Final = candidate.request("POST", "/v1/messages", _body(model, prompt, "deployment")) + _assert_stream_failed_after_message_start(response.status_code, parse_sse(response.text), wire, attempts=2) + delivered: Final = eventually( + _SinkReader(sink, model).delivered_usage, lambda values: len(values) >= 2, seconds=20 + ) + assert delivered == (("failure", 5, 1), ("failure", 5, 1)), delivered + rows: Final = eventually( + lambda: _request_rows(candidate, response.headers["x-litellm-call-id"]), + lambda values: len(values) >= 1, + seconds=70, + ) + assert rows == [ + { + "status": "failure", + "prompt_tokens": sum(input_tokens for _, input_tokens, _ in delivered), + "completion_tokens": sum(output_tokens for _, _, output_tokens in delivered), + } + ], rows + + def _string_values(cache: Redis) -> tuple[bytes, ...]: keys: Final = tuple(key for key in cache.scan_iter(count=1000) if cache.type(key) == b"string") return tuple(value for value in cache.mget(keys) if value is not None) if keys else () @@ -485,7 +614,7 @@ def test_retried_stream_is_cached_and_the_identical_request_is_served_without_th model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_API_KEY, num_retries=1) body: Final = _body(model, prompt, "deployment") status, events = _stream(gateway, body) - _assert_completed_by_retry(status, events, prompt, model, wire) + _assert_completed_by_retry(gateway, status, events, prompt, model, wire) eventually(lambda: _cached_somewhere(_served_id(prompt, 2)), bool, seconds=30) replay_status, replay = _stream(gateway, body) assert replay_status == 200, replay diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index 43190852c60..bbfae2229ce 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -1849,13 +1849,19 @@ def test_mark_logging_complete_flags_by_stream_and_event( "async_failure", "sync_failure", ) + attempt_marker: Final = datetime_unit_test(2026, 1, 1) logging_obj.stream = stream + logging_obj.model_call_details["current_attempt_api_call_start_time"] = attempt_marker logging_obj.mark_logging_complete(event_type=event) expected_to_run: Final = stream and event in ("async_success", "sync_success") assert logging_obj.should_run_logging(event_type=event) is expected_to_run assert all(logging_obj.should_run_logging(event_type=other_event) for other_event in events if other_event != event) + if event == "async_failure": + assert logging_obj.model_call_details["async_failure_logged_attempt"] is attempt_marker + else: + assert "async_failure_logged_attempt" not in logging_obj.model_call_details @pytest.mark.parametrize("stream", [False, True]) diff --git a/tests/unit/litellm_core_utils/test_max_streaming_duration.py b/tests/unit/litellm_core_utils/test_max_streaming_duration.py index fc8daab3899..234bc83e3c3 100644 --- a/tests/unit/litellm_core_utils/test_max_streaming_duration.py +++ b/tests/unit/litellm_core_utils/test_max_streaming_duration.py @@ -67,12 +67,13 @@ class TestCustomStreamWrapperMaxDuration: @pytest.mark.asyncio async def test_should_raise_on_async_anext_when_exceeded(self): - """__anext__ should check the limit before iterating, dispatching the - same failure-callback/logging path every other stream failure goes - through (dispatch_failure_handlers is async on the real Logging class, - so the mock needs to be awaitable too).""" + """__anext__ checks the limit before iterating and schedules failure logging.""" wrapper = _make_custom_stream_wrapper() - wrapper.logging_obj.dispatch_failure_handlers = AsyncMock() + + async def _noop() -> None: + return None + + wrapper.logging_obj.dispatch_attempt_failure_handlers = MagicMock(side_effect=lambda *_: _noop()) wrapper._stream_created_time = time.time() - 20 with patch("litellm.constants.LITELLM_MAX_STREAMING_DURATION_SECONDS", 10.0): with pytest.raises(litellm.Timeout): diff --git a/tests/unit/litellm_core_utils/test_streaming_handler.py b/tests/unit/litellm_core_utils/test_streaming_handler.py index 550ecd3c0f7..c13cea51183 100644 --- a/tests/unit/litellm_core_utils/test_streaming_handler.py +++ b/tests/unit/litellm_core_utils/test_streaming_handler.py @@ -13,8 +13,7 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest from openai.types.chat import ChatCompletionChunk -from pydantic import BaseModel - +from pydantic import BaseModel, TypeAdapter import litellm from litellm import verbose_logger from litellm._logging import session_id_var, trace_id_var @@ -36,6 +35,7 @@ from litellm.types.utils import ( ModelResponseStream, PromptTokensDetailsWrapper, StreamingChoices, + StandardLoggingPayload, Usage, ) from litellm.utils import ModelResponseListIterator @@ -4265,6 +4265,68 @@ def test_record_partial_usage_for_failure_stashes_usage_and_cost(): assert isinstance(logging_obj.model_call_details["response_cost"], float) +class _FailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.failures: tuple[tuple[int, int], ...] = () + self.logged: asyncio.Event = asyncio.Event() + + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + payload: Final = TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"]) + self.failures = (*self.failures, (payload["prompt_tokens"], payload["completion_tokens"])) + self.logged.set() + + +@pytest.mark.asyncio +async def test_log_stream_failure_bills_the_dropped_attempt_even_after_the_router_discards_its_usage() -> None: + """The dropped attempt's failure log is queued, and the router discards the stashed usage as it opens the + fallback hop before that log runs. The log must still bill what the dropped attempt consumed, and the shared + logging object must not be marked as already failed, since the hop's own failure still has to reach the + integrations.""" + recorder: Final = _FailureRecorder() + logging_obj: Final = Logging( + model="gpt-5.6", + messages=[{"role": "user", "content": "Hey"}], + stream=True, + call_type="completion", + start_time=datetime.datetime(2026, 1, 1), + litellm_call_id="partial-usage-copy", + function_id="1245", + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model="gpt-5.6", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + wrapper.chunks = [ + ModelResponseStream( + id="chatcmpl-partial-copy", + created=1742056047, + model="gpt-5.6", + object="chat.completion.chunk", + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="The", role="assistant"))], + usage=Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31), + ) + ] + + with pytest.raises(MidStreamFallbackError): + wrapper._log_stream_failure_and_raise(httpx.ReadError("connection dropped")) + logging_obj.carry_failed_attempt_usage() + await asyncio.wait_for(recorder.logged.wait(), timeout=10) + + assert recorder.failures == ((30, 1),) + assert logging_obj.model_call_details.get("has_logged_async_failure") is not True + + def test_record_partial_usage_for_failure_noop_without_chunks(): """With no chunks delivered there is nothing billed to recover, so the failure stash must stay absent and not force a zero-usage row. diff --git a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py b/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py index 785e1f1096b..6951253349e 100644 --- a/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py +++ b/tests/unit/llms/anthropic/pass_through/adapters/test_streaming_iterator_mid_stream_error.py @@ -19,12 +19,14 @@ import json import os import sys import threading +from collections.abc import Callable, Mapping, Sequence from datetime import datetime -from collections.abc import Callable -from typing import List, Optional +from typing import Final, Iterator, List, Optional from unittest.mock import MagicMock +import httpx import pytest +from pydantic import TypeAdapter import litellm @@ -33,12 +35,13 @@ sys.path.insert(0, os.path.abspath("../../../../..")) from litellm.exceptions import MidStreamFallbackError from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm.llms.anthropic.pass_through.adapters.streaming_iterator import ( AnthropicStreamWrapper, _mid_stream_error_sse_event, ) from litellm.llms.bedrock.common_utils import BedrockError -from litellm.types.utils import Delta, StreamingChoices +from litellm.types.utils import Delta, ModelResponseStream, StandardLoggingPayload, StreamingChoices, Usage def _make_chunk(delta: Delta, finish_reason: Optional[str] = None) -> MagicMock: @@ -55,14 +58,14 @@ class _AsyncStreamThenRaise: """Yields the given chunks, then raises ``exc`` (mimics a provider stream that terminates mid-response).""" - def __init__(self, items: List[MagicMock], exc: BaseException): + def __init__(self, items: Sequence[object], exc: BaseException) -> None: self._it = iter(items) self._exc = exc - def __aiter__(self): + def __aiter__(self) -> "_AsyncStreamThenRaise": return self - async def __anext__(self): + async def __anext__(self) -> object: try: return next(self._it) except StopIteration: @@ -182,9 +185,73 @@ class _AsyncFailureRecorder(CustomLogger): def __init__(self): super().__init__() self.exceptions: list[BaseException] = [] + self.usages: tuple[tuple[int, int], ...] = () - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + ) -> None: self.exceptions.append(kwargs["exception"]) + payload: Final = TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"]) + self.usages = (*self.usages, (payload["prompt_tokens"], payload["completion_tokens"])) + + +class _BoundaryAsyncFailureRecorder(_AsyncFailureRecorder): + def __init__(self) -> None: + super().__init__() + self.started = 0 + self.completed = 0 + + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime, + end_time: datetime, + ) -> None: + self.started += 1 + await asyncio.sleep(0.05) + await super().async_log_failure_event(kwargs, response_obj, start_time, end_time) + self.completed += 1 + + +def test_anthropic_adapter_does_not_repeat_custom_stream_failure_dispatch() -> None: + async_recorder: Final = _BoundaryAsyncFailureRecorder() + sync_recorder: Final = _SyncFailureRecorder() + logging_obj: Final = _make_logging_obj("wrapped-custom-stream", async_recorder, sync_recorder) + chunk: Final = ModelResponseStream( + id="chatcmpl-dropped", + object="chat.completion.chunk", + created=1, + model="gpt-5.6", + choices=[StreamingChoices(index=0, delta=Delta(content="partial"), finish_reason=None)], + usage=Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31), + ) + inner_stream: Final = CustomStreamWrapper( + completion_stream=_AsyncStreamThenRaise([chunk], httpx.ReadError("stream dropped")), + model="gpt-5.6", + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + wrapper: Final = AnthropicStreamWrapper( + completion_stream=inner_stream, + model="openai/gpt-5.6", + litellm_logging_obj=logging_obj, + ) + + events: Final = asyncio.run(_drain_sse(wrapper)) + + assert async_recorder.started == 1 + assert async_recorder.completed == 1 + assert sync_recorder.called.wait(timeout=5), "sync failure callback never ran" + assert _parse_sse(events[-1])[0] == "error" + assert len(async_recorder.exceptions) == 1 + assert len(sync_recorder.exceptions) == 1 + assert async_recorder.usages == ((30, 1),) + assert sync_recorder.exceptions[0] == async_recorder.exceptions[0] class _SyncFailureRecorder: diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index d2b5c8d5bd9..4c3ee9e7ca5 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -3278,6 +3278,14 @@ async def test_vertex_ai_streaming_bad_request_is_not_wrapped(): async def dispatch_failure_handlers(self, *args, **kwargs): return None + async def dispatch_attempt_failure_handlers( + self, exception: Exception, traceback_exception: str + ) -> None: + return None + + def attempt_scoped_copy(self) -> "DummyLogging": + return self + async def failing_make_call(client=None, **kwargs): raise VertexAIError(status_code=400, message="bad input", headers={}) diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index 4770ecd5485..cc701e1ebd4 100644 --- a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -2643,57 +2643,6 @@ class TestRecordPartialUsageForFailure: assert "response_cost" not in logging_obj.model_call_details -class _FailureRecorder(CustomLogger): - def __init__(self): - super().__init__() - self.failure_kwargs = [] - - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): - self.failure_kwargs.append(kwargs) - - -class TestLogErrorFrameAsFailure: - """A stream closed at the provider's `event: error` frame is one failed attempt: its failure log bills the - usage streamed before the frame on the attempt-scoped copy its caller hands it, and the request's shared - logging state stays as it was, so the attempt the router opens next logs with nothing left behind.""" - - @pytest.mark.asyncio - async def test_logs_a_failure_with_the_usage_streamed_before_the_frame_and_leaves_the_request_state_alone(self): - recorder = _FailureRecorder() - logging_obj = LiteLLMLoggingObj( - model="claude-sonnet-5", - messages=[{"role": "user", "content": "hello"}], - stream=True, - call_type="anthropic_messages", - start_time=datetime.now(), - litellm_call_id="test-error-frame-failure", - function_id="test-error-frame-failure", - dynamic_async_failure_callbacks=[recorder], - ) - chunks = [ - *TestRecordPartialUsageForFailure()._interrupted_chunks()[:1], - TestRecordPartialUsageForFailure._sse( - "error", {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}} - ), - ] - - await AnthropicPassthroughLoggingHandler.log_error_frame_as_failure( - attempt_logging_obj=logging_obj.attempt_scoped_copy(), - request_body={"model": "claude-sonnet-5", "stream": True}, - all_chunks=chunks, - error_event=("overloaded_error", "Overloaded", 529), - ) - - assert len(recorder.failure_kwargs) == 1 - failure_payload = recorder.failure_kwargs[0]["standard_logging_object"] - assert failure_payload["status"] == "failure" - assert failure_payload["prompt_tokens"] == 52 - assert failure_payload["response_cost"] > 0 - assert isinstance(recorder.failure_kwargs[0]["exception"], litellm.InternalServerError) - assert "combined_usage_object" not in logging_obj.model_call_details - assert "response_cost" not in logging_obj.model_call_details - assert logging_obj.should_run_logging("async_failure") - @pytest.mark.parametrize( ("all_chunks", "interrupted"), diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py index da084146894..7d56487a844 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler.py @@ -220,7 +220,11 @@ async def test_failed_anthropic_stream_records_partial_usage_off_the_event_loop( logging_obj.model_call_details = {"model": model, "stream": True} logging_obj.litellm_params = {} logging_obj.get_router_model_id.return_value = None + logging_obj.attempt_scoped_copy.return_value = logging_obj logging_obj.dispatch_failure_handlers = AsyncMock() + logging_obj.dispatch_attempt_failure_handlers = AsyncMock( + side_effect=logging_obj.dispatch_failure_handlers + ) _, took, lags = await timed_with_loop_lags( lambda: PassThroughStreamingHandler.schedule_stream_failure_logging( diff --git a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py index 8ce41129aec..4aa7e9cfbae 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py +++ b/tests/unit/proxy/pass_through_endpoints/test_streaming_handler_interrupt.py @@ -2,7 +2,9 @@ import asyncio import json +from collections.abc import AsyncGenerator, AsyncIterator, Mapping, Sequence from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -22,6 +24,8 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import aclo from litellm.proxy.pass_through_endpoints.streaming_handler import ( PassThroughStreamingHandler, ) +from litellm.proxy.utils import _failure_fields_to_lift +from litellm.router_utils.fallback_event_handlers import carry_superseded_attempt_usage from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType @@ -407,7 +411,7 @@ async def _collect_openai_passthrough_chunks(chunks, endpoint_type): request_body={"model": "gpt-4o-mini", "stream": True}, litellm_logging_obj=_unarmed_logging_obj(), endpoint_type=endpoint_type, - start_time=datetime.now(), + start_time=datetime(2026, 1, 1), passthrough_success_handler_obj=MagicMock(), url_route="/openai/v1/chat/completions", route_streaming_logging=AsyncMock(), @@ -705,7 +709,7 @@ async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception( messages=[{"role": "user", "content": "hi"}], stream=True, call_type="anthropic_messages", - start_time=datetime.now(), + start_time=datetime(2026, 1, 1), litellm_call_id="test-mid-stream-timeout", function_id="test-mid-stream-timeout", dynamic_async_success_callbacks=[recorder], @@ -750,6 +754,109 @@ async def test_chunk_processor_logs_failure_not_success_on_mid_stream_exception( assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadTimeout) +def _anthropic_stream_dropping_after(frames: tuple[bytes, ...]) -> httpx.Response: + mock: Final = MagicMock(spec=httpx.Response) + mock.status_code = 200 + + async def _aiter_bytes() -> AsyncIterator[bytes]: + for frame in frames: + yield frame + raise httpx.ReadError("upstream closed the connection") + + mock.aiter_bytes = _aiter_bytes + return mock + + +def _anthropic_messages_attempt( + logging_obj: LiteLLMLoggingObj, response: httpx.Response +) -> AsyncGenerator[bytes, None]: + return PassThroughStreamingHandler.chunk_processor( + response=response, + request_body={"model": "claude-sonnet-5", "stream": True}, + litellm_logging_obj=logging_obj, + endpoint_type=EndpointType.ANTHROPIC, + start_time=datetime(2026, 1, 1), + passthrough_success_handler_obj=MagicMock(), + url_route="/v1/messages", + ) + + +def _anthropic_messages_logging_obj(recorder: _EventRecorder, call_id: str) -> LiteLLMLoggingObj: + return LiteLLMLoggingObj( + model="claude-sonnet-5", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime(2026, 1, 1), + litellm_call_id=call_id, + function_id=call_id, + dynamic_async_success_callbacks=[recorder], + dynamic_async_failure_callbacks=[recorder], + ) + + +async def _drop_after_message_start(logging_obj: LiteLLMLoggingObj, message_id: str, input_tokens: int) -> None: + dropped: Final = _anthropic_messages_attempt( + logging_obj, _anthropic_stream_dropping_after((_anthropic_message_start_frame(message_id, input_tokens),)) + ) + with pytest.raises(httpx.ReadError): + async for _ in dropped: + pass + + +@pytest.mark.asyncio +async def test_dropped_attempt_logs_its_failure_without_claiming_the_shared_logging_object() -> None: + """The router retries a /v1/messages stream the provider drops after message_start on the same logging + object. The dropped attempt's failure log bills its own usage and leaves the shared object free: the + retry's success log carries the retry's usage and no stale exception, while the stash stays for the + failure row of a request that runs out of retries until the router opens the next attempt.""" + recorder: Final = _EventRecorder() + logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-dropped-attempt") + + await _drop_after_message_start(logging_obj, "msg_a1", 52) + await _settle(recorder.failure_kwargs) + + assert recorder.failure_kwargs[0]["standard_logging_object"]["prompt_tokens"] == 52 + assert isinstance(recorder.failure_kwargs[0]["exception"], httpx.ReadError) + assert logging_obj.model_call_details.get("has_logged_async_failure") is not True + assert "exception" not in logging_obj.model_call_details + stashed: Final = logging_obj.model_call_details.get("combined_usage_object") + assert isinstance(stashed, litellm.Usage) and stashed.prompt_tokens == 52 + + logging_obj.carry_failed_attempt_usage() + retry: Final = _anthropic_messages_attempt( + logging_obj, _anthropic_stream_of((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail())) + ) + async for _ in retry: + pass + await _settle(recorder.success_kwargs) + + success_payload: Final = recorder.success_kwargs[0]["standard_logging_object"] + assert success_payload["status"] == "success" + assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3) + assert recorder.success_kwargs[0].get("exception") is None + assert len(recorder.failure_kwargs) == 1 + + +@pytest.mark.asyncio +async def test_every_dropped_attempt_logs_its_own_failure() -> None: + """Two attempts of one /v1/messages request dropped in a row each log a failure billing their own usage; + the first attempt's log must not swallow the second's, and the shared object ends up holding the last + attempt's usage for the request's failure row.""" + recorder: Final = _EventRecorder() + logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-every-dropped-attempt") + + await _drop_after_message_start(logging_obj, "msg_a1", 52) + await _settle(recorder.failure_kwargs) + logging_obj.carry_failed_attempt_usage() + await _drop_after_message_start(logging_obj, "msg_a2", 9) + await _settle(recorder.failure_kwargs, count=2) + + assert [kwargs["standard_logging_object"]["prompt_tokens"] for kwargs in recorder.failure_kwargs] == [52, 9] + assert all(isinstance(kwargs["exception"], httpx.ReadError) for kwargs in recorder.failure_kwargs) + stashed: Final = logging_obj.model_call_details.get("combined_usage_object") + assert isinstance(stashed, litellm.Usage) and stashed.prompt_tokens == 9 + def _anthropic_message_start_frame(message_id: str = "msg_1", input_tokens: int = 52) -> bytes: return _anthropic_sse( "message_start", @@ -788,9 +895,9 @@ def _anthropic_completed_stream_tail() -> tuple[bytes, ...]: ) -async def _settle(recorded: list[dict[str, object]]) -> None: +async def _settle(recorded: Sequence[Mapping[str, object]], count: int = 1) -> None: for _ in range(300): - if recorded: + if len(recorded) >= count: return await asyncio.sleep(0.01) @@ -857,8 +964,6 @@ async def test_chunk_processor_logs_a_provider_error_frame_as_a_failure_when_clo assert received == list(frames) assert recorder.success_kwargs == [] assert logging_obj.model_call_details.get("has_dispatched_final_stream_success") is not True - assert "combined_usage_object" not in logging_obj.model_call_details - assert "response_cost" not in logging_obj.model_call_details assert logging_obj.should_run_logging("async_failure") assert len(recorder.failure_kwargs) == 1 failure_payload = recorder.failure_kwargs[0]["standard_logging_object"] @@ -991,6 +1096,31 @@ async def test_superseded_attempt_failure_log_keeps_its_own_deployment_when_the_ assert logging_obj.model_call_details["litellm_params"]["model_info"]["id"] == "deployment-2" +@pytest.mark.asyncio +async def test_error_frame_then_exhausted_retry_bills_both_attempts() -> None: + recorder: Final = _EventRecorder() + logging_obj: Final = _anthropic_messages_logging_obj(recorder, "test-error-frame-terminal-billing") + superseded: Final = _anthropic_messages_attempt( + logging_obj, _anthropic_stream_of((_anthropic_message_start_frame("msg_a1", 52), _anthropic_overloaded_error_frame())) + ) + async for chunk in superseded: + if b"event: error" in chunk: + break + await aclose_superseded_attempt(superseded) + carry_superseded_attempt_usage({"litellm_logging_obj": logging_obj}) + await _drop_after_message_start(logging_obj, "msg_a2", 9) + await _settle(recorder.failure_kwargs, count=2) + + failures: Final = tuple(event["standard_logging_object"] for event in recorder.failure_kwargs) + assert tuple(event["prompt_tokens"] for event in failures) == (52, 9) + terminal: Final = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}) + usage: Final = terminal["combined_usage_object"] + assert isinstance(usage, litellm.Usage) + assert usage.prompt_tokens == sum(event["prompt_tokens"] for event in failures) + assert usage.completion_tokens == sum(event["completion_tokens"] for event in failures) + assert terminal["response_cost"] == pytest.approx(sum(event["response_cost"] for event in failures)) + + @pytest.mark.asyncio async def test_retry_success_log_carries_its_own_usage_after_a_provider_error_frame(): """Both attempts of a retried /v1/messages stream log through the request's one logging object. The @@ -1025,6 +1155,7 @@ async def test_retry_success_log_carries_its_own_usage_after_a_provider_error_fr if b"event: error" in chunk: break await aclose_superseded_attempt(superseded) + carry_superseded_attempt_usage({"litellm_logging_obj": logging_obj}) await _settle(recorder.failure_kwargs) retry = attempt((_anthropic_message_start_frame("msg_a2", 7), *_anthropic_completed_stream_tail())) async for _ in retry: @@ -1036,8 +1167,6 @@ async def test_retry_success_log_carries_its_own_usage_after_a_provider_error_fr success_payload = recorder.success_kwargs[0]["standard_logging_object"] assert success_payload["status"] == "success" assert (success_payload["prompt_tokens"], success_payload["completion_tokens"]) == (7, 3) - - def _google_sse(prompt_tokens: int, completion_tokens: int, text: str) -> bytes: payload = { "candidates": [{"content": {"parts": [{"text": text}], "role": "model"}, "index": 0}], diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 7b18cb3350c..ab6a874a010 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -1,5 +1,8 @@ +import asyncio import datetime as real_datetime import smtplib +import traceback +from collections.abc import Mapping from typing import Final import httpx @@ -11,8 +14,15 @@ from litellm.exceptions import InternalServerError from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.bug_report import ISSUE_URL_BASE +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy._types import ProxyErrorTypes, UserAPIKeyAuth -from litellm.proxy.utils import PrismaClient, ProxyLogging, handle_exception_on_proxy +from litellm.proxy.utils import ( + PrismaClient, + ProxyLogging, + _failure_fields_to_lift, + handle_exception_on_proxy, +) +from litellm.router_utils.fallback_event_handlers import carry_superseded_attempt_usage from litellm.types.guardrails import GuardrailEventHooks @@ -2472,10 +2482,681 @@ def test_attempts_already_summed_into_the_final_row_are_not_lifted_again(): lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) - assert lifted["combined_usage_object"].prompt_tokens == 24 + current_only_logging_obj: Final = _dispatched_failure_logging_obj( + FAILURE_USAGE_MODEL, + get_litellm_params( + metadata={ + "attempted_retries": 1, + "model_info": {"id": "dep-lit145-final"}, + "previous_models": ( + {"deployment_id": "dep-lit145-final", "exception_type": "Timeout", "attempted_retries": 0}, + ), + } + ), + ) + current_only: Final = _failure_fields_to_lift( + {"litellm_logging_obj": current_only_logging_obj}, provider_billed=True + ) + current_only_usage: Final = current_only["combined_usage_object"] + + assert current_only_usage.prompt_tokens > 0 + assert lifted["combined_usage_object"].prompt_tokens == 24 + current_only_usage.prompt_tokens assert "billed_failed_attempts" not in lifted +def test_carry_superseded_attempt_usage_without_stashed_usage_preserves_failure_estimate() -> None: + request_start: Final = real_datetime.datetime(2026, 1, 1) + logging_obj: Final = Logging( + model=FAILURE_USAGE_MODEL, + messages=ONE_USER_MESSAGE, + stream=False, + call_type="acompletion", + start_time=request_start, + litellm_call_id="call-lit145-no-stashed-usage", + function_id="fn-lit145-no-stashed-usage", + ) + logging_obj.pre_call(input=ONE_USER_MESSAGE, api_key="sk-test", model=FAILURE_USAGE_MODEL) + logging_obj.model_call_details["call_type"] = "acompletion" + logging_obj.model_call_details["messages"] = ONE_USER_MESSAGE + + try: + try: + raise httpx.ReadTimeout("no bytes in time") + except httpx.ReadTimeout as cause: + raise litellm.Timeout("Request timed out.", model=FAILURE_USAGE_MODEL, llm_provider="openai") from cause + except litellm.Timeout as timeout: + logging_obj.failure_handler( + timeout, + traceback.format_exc(), + request_start, + request_start + real_datetime.timedelta(seconds=1), + ) + + assert "combined_usage_object" not in logging_obj.model_call_details + request_data: Final = {"litellm_logging_obj": logging_obj} + before: Final = _failure_fields_to_lift(request_data, provider_billed=True) + + carry_superseded_attempt_usage(request_data) + + after: Final = _failure_fields_to_lift(request_data, provider_billed=True) + before_usage: Final = before.get("combined_usage_object") + after_usage: Final = after.get("combined_usage_object") + assert before_usage is not None + assert after_usage is not None + assert after_usage == before_usage + assert before["response_cost"] > 0 + assert after["response_cost"] == before["response_cost"] + + +@pytest.mark.asyncio +async def test_dropped_stream_failure_attribution_reaches_terminal_spend_row() -> None: + from litellm.exceptions import MidStreamFallbackError + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices, Usage + + request_start: Final = real_datetime.datetime(2026, 1, 1) + model: Final = "deployment-model" + logging_obj: Final = Logging( + model=model, + messages=ONE_USER_MESSAGE, + stream=True, + call_type="acompletion", + start_time=request_start, + litellm_call_id="call-lit145-attribution", + function_id="fn-lit145-attribution", + ) + logging_obj.update_environment_variables( + model=model, + user="", + optional_params={}, + litellm_params=get_litellm_params(metadata={"model_info": {"id": "deployment-id"}}), + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "custom_llm_provider": "openai", + "first_api_call_start_time": request_start, + "current_attempt_api_call_start_time": request_start, + } + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model=model, + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + wrapper.chunks = [ + ModelResponseStream( + id="chatcmpl-attribution", + created=1742056047, + model=model, + object="chat.completion.chunk", + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="The"))], + usage=Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31), + ) + ] + exception: Final = httpx.ReadError("connection dropped") + + with pytest.raises(MidStreamFallbackError): + wrapper._log_stream_failure_and_raise(exception) + await wrapper.wait_for_failure_logging(timeout=10) + + request_data: Final = {"litellm_logging_obj": logging_obj} + lifted: Final = _failure_fields_to_lift(request_data, provider_billed=True) + terminal_kwargs: Final = { + key: value + for key, value in {**request_data, **lifted}.items() + if key != "litellm_logging_obj" + } + row: Final = get_logging_payload( + kwargs=terminal_kwargs, + response_obj=exception, + start_time=request_start, + end_time=request_start + real_datetime.timedelta(seconds=1), + ) + + assert row["model_id"] == "deployment-id" + assert row["custom_llm_provider"] == "openai" + assert row["model"] == model + + +class _ReplaceSharedFailurePayload(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.logging_obj: Logging | None = None + self.replaced: asyncio.Event = asyncio.Event() + self.started: asyncio.Event = asyncio.Event() + self.release: asyncio.Event = asyncio.Event() + self.pause_until_released: bool = False + self.replacement_payload: object | None = None + + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: real_datetime.datetime, + end_time: real_datetime.datetime, + ) -> None: + from pydantic import TypeAdapter + + from litellm.types.utils import StandardLoggingPayload + + payload: Final = TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"]) + self.started.set() + if self.pause_until_released: + await self.release.wait() + logging_obj: Final = self.logging_obj + if logging_obj is None: + return + replacement: Final = self.replacement_payload or TypeAdapter(StandardLoggingPayload).validate_python( + { + **payload, + "model_id": "newer-deployment-id", + "custom_llm_provider": "newer-provider", + "model": "newer-model", + } + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "standard_logging_object": replacement, + } + self.replaced.set() + + +class _OrderedAttemptFailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.attempt_a_started: asyncio.Event = asyncio.Event() + self.attempt_b_started: asyncio.Event = asyncio.Event() + self.release_attempt_a: asyncio.Event = asyncio.Event() + self.release_attempt_b: asyncio.Event = asyncio.Event() + + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: real_datetime.datetime, + end_time: real_datetime.datetime, + ) -> None: + from pydantic import TypeAdapter + + from litellm.types.utils import StandardLoggingPayload + + payload: Final = TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"]) + if payload["model_id"] == "deployment-a": + self.attempt_a_started.set() + await self.release_attempt_a.wait() + return + if payload["model_id"] == "deployment-b": + self.attempt_b_started.set() + await self.release_attempt_b.wait() + + +class _PausedBaselineFailureRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.started: asyncio.Event = asyncio.Event() + self.release: asyncio.Event = asyncio.Event() + self.standard_logging_object: object | None = None + + async def async_log_failure_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: real_datetime.datetime, + end_time: real_datetime.datetime, + ) -> None: + self.standard_logging_object = kwargs.get("standard_logging_object") + self.started.set() + await self.release.wait() + + +@pytest.mark.asyncio +async def test_latest_dropped_attempt_publishes_its_payload_after_earlier_dispatch_finishes() -> None: + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + from litellm.types.utils import Usage + + request_start: Final = real_datetime.datetime(2026, 1, 1) + marker_a: Final = request_start + marker_b: Final = request_start + real_datetime.timedelta(seconds=1) + recorder: Final = _OrderedAttemptFailureRecorder() + logging_obj: Final = Logging( + model="deployment-model-a", + messages=ONE_USER_MESSAGE, + stream=True, + call_type="acompletion", + start_time=request_start, + litellm_call_id="call-lit145-attempt-order", + function_id="fn-lit145-attempt-order", + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="deployment-model-a", + user="", + optional_params={}, + litellm_params=get_litellm_params(metadata={"model_info": {"id": "deployment-a"}}), + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "custom_llm_provider": "openai", + "first_api_call_start_time": request_start, + "current_attempt_api_call_start_time": marker_a, + "combined_usage_object": Usage(prompt_tokens=4, completion_tokens=1, total_tokens=5), + "response_cost": 0.001, + } + + attempt_a: Final = asyncio.create_task( + logging_obj.dispatch_attempt_failure_handlers(httpx.ReadError("attempt A dropped"), "") + ) + await asyncio.wait_for(recorder.attempt_a_started.wait(), timeout=10) + + logging_obj.carry_failed_attempt_usage() + logging_obj.update_environment_variables( + model="deployment-model-b", + user="", + optional_params={}, + litellm_params=get_litellm_params(metadata={"model_info": {"id": "deployment-b"}}), + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "model": "deployment-model-b", + "custom_llm_provider": "anthropic", + "current_attempt_api_call_start_time": marker_b, + "combined_usage_object": Usage(prompt_tokens=7, completion_tokens=2, total_tokens=9), + "response_cost": 0.002, + } + + attempt_b: Final = asyncio.create_task( + logging_obj.dispatch_attempt_failure_handlers(httpx.ReadError("attempt B dropped"), "") + ) + await asyncio.wait_for(recorder.attempt_b_started.wait(), timeout=10) + recorder.release_attempt_a.set() + await attempt_a + recorder.release_attempt_b.set() + await attempt_b + + request_data: Final = {"litellm_logging_obj": logging_obj} + lifted: Final = _failure_fields_to_lift(request_data, provider_billed=True) + terminal_kwargs: Final = { + key: value + for key, value in {**request_data, **lifted}.items() + if key != "litellm_logging_obj" + } + row: Final = get_logging_payload( + kwargs=terminal_kwargs, + response_obj=httpx.ReadError("attempt B dropped"), + start_time=request_start, + end_time=request_start + real_datetime.timedelta(seconds=1), + ) + + assert row["model_id"] == "deployment-b" + assert row["custom_llm_provider"] == "anthropic" + assert row["model"] == "deployment-model-b" + + +@pytest.mark.asyncio +async def test_dropped_attempt_attribution_is_published_while_failure_callback_is_paused() -> None: + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + from litellm.types.utils import Usage + + request_start: Final = real_datetime.datetime(2026, 1, 1) + marker_a: Final = request_start + failure: Final = httpx.ReadError("attempt A dropped") + recorder: Final = _OrderedAttemptFailureRecorder() + logging_obj: Final = Logging( + model="deployment-model-a", + messages=ONE_USER_MESSAGE, + stream=True, + call_type="acompletion", + start_time=request_start, + litellm_call_id="call-lit145-attempt-paused", + function_id="fn-lit145-attempt-paused", + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="deployment-model-a", + user="", + optional_params={}, + litellm_params=get_litellm_params(metadata={"model_info": {"id": "deployment-a"}}), + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "custom_llm_provider": "openai", + "first_api_call_start_time": request_start, + "current_attempt_api_call_start_time": marker_a, + "combined_usage_object": Usage(prompt_tokens=4, completion_tokens=1, total_tokens=5), + "response_cost": 0.001, + } + + attempt: Final = asyncio.create_task(logging_obj.dispatch_attempt_failure_handlers(failure, "")) + await asyncio.wait_for(recorder.attempt_a_started.wait(), timeout=10) + + request_data: Final = {"litellm_logging_obj": logging_obj} + lifted: Final = _failure_fields_to_lift(request_data, provider_billed=True) + terminal_kwargs: Final = { + key: value + for key, value in {**request_data, **lifted}.items() + if key != "litellm_logging_obj" + } + row: Final = get_logging_payload( + kwargs=terminal_kwargs, + response_obj=failure, + start_time=request_start, + end_time=request_start + real_datetime.timedelta(seconds=1), + ) + + try: + assert row["model_id"] == "deployment-a" + assert row["custom_llm_provider"] == "openai" + assert row["model"] == "deployment-model-a" + assert row["prompt_tokens"] == 4 + assert row["completion_tokens"] == 1 + finally: + recorder.release_attempt_a.set() + await attempt + + +@pytest.mark.asyncio +async def test_failed_attempt_publishes_baseline_reason_after_invalidation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from pydantic import TypeAdapter + + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.hooks.autorouter_baseline_cache import AutoRouterBaselineCache, CapturedBaselineObservation + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + from litellm.router import Router + from litellm.types.utils import CallTypes, StandardLoggingPayload, Usage + from tests.unit.proxy.hooks.test_autorouter_baseline_cache import _MODELS, _kwargs + + request_start: Final = real_datetime.datetime(2026, 1, 1) + marker: Final = request_start + model: Final = "deployment-model-a" + router: Final = Router(model_list=_MODELS, disable_cooldowns=True) + baseline_cache: Final = AutoRouterBaselineCache( + None, + router=lambda: router, + clock=lambda: request_start.timestamp() + 10, + ) + recorder: Final = _PausedBaselineFailureRecorder() + logging_obj: Final = Logging( + model=model, + messages=ONE_USER_MESSAGE, + stream=True, + call_type=CallTypes.anthropic_messages.value, + start_time=request_start, + litellm_call_id="call-lit145-baseline-failure", + function_id="fn-lit145-baseline-failure", + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model=model, + user="", + optional_params={}, + litellm_params=get_litellm_params(metadata={"model_info": {"id": "deployment-a"}}), + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "custom_llm_provider": "openai", + "first_api_call_start_time": request_start, + "current_attempt_api_call_start_time": marker, + "combined_usage_object": Usage(prompt_tokens=4, completion_tokens=1, total_tokens=5), + "response_cost": 0.001, + } + await baseline_cache.async_pre_call_deployment_hook(_kwargs(logging_obj), CallTypes.anthropic_messages) + assert logging_obj.baseline_cache_context is not None + + failure: Final = httpx.ReadError("attempt A dropped") + attempt: Final = asyncio.create_task(logging_obj.dispatch_attempt_failure_handlers(failure, "")) + try: + await asyncio.wait_for(recorder.started.wait(), timeout=10) + request_data: Final = {"litellm_logging_obj": logging_obj} + lifted: Final = _failure_fields_to_lift(request_data, provider_billed=True) + terminal_kwargs: Final = { + key: value + for key, value in {**request_data, **lifted}.items() + if key != "litellm_logging_obj" + } + row: Final = get_logging_payload( + kwargs=terminal_kwargs, + response_obj=failure, + start_time=request_start, + end_time=request_start + real_datetime.timedelta(seconds=1), + ) + + assert row["model_id"] == "deployment-a" + assert row["custom_llm_provider"] == "openai" + assert row["model"] == model + assert row["prompt_tokens"] == 4 + assert row["completion_tokens"] == 1 + lifted_payload: Final = TypeAdapter(StandardLoggingPayload).validate_python(lifted["standard_logging_object"]) + callback_payload: Final = TypeAdapter(StandardLoggingPayload).validate_python( + recorder.standard_logging_object + ) + + def baseline_reason(payload: StandardLoggingPayload) -> str: + encoded: Final = payload["autorouter_baseline_observation"] + assert isinstance(encoded, str) + return CapturedBaselineObservation.model_validate_json(encoded).observation.reason or "" + + assert baseline_reason(lifted_payload) == "failed_request" + assert baseline_reason(callback_payload) == "failed_request" + finally: + recorder.release.set() + await attempt + + +@pytest.mark.asyncio +async def test_queued_stale_attempt_dispatch_does_not_publish_over_current_attempt() -> None: + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + from litellm.types.utils import Usage + + request_start: Final = real_datetime.datetime(2026, 1, 1) + marker_a: Final = request_start + marker_b: Final = request_start + real_datetime.timedelta(seconds=1) + recorder: Final = _OrderedAttemptFailureRecorder() + logging_obj: Final = Logging( + model="deployment-model-a", + messages=ONE_USER_MESSAGE, + stream=True, + call_type="acompletion", + start_time=request_start, + litellm_call_id="call-lit145-stale-attempt", + function_id="fn-lit145-stale-attempt", + dynamic_async_failure_callbacks=[recorder], + ) + logging_obj.update_environment_variables( + model="deployment-model-a", + user="", + optional_params={}, + litellm_params=get_litellm_params(metadata={"model_info": {"id": "deployment-a"}}), + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "custom_llm_provider": "openai", + "first_api_call_start_time": request_start, + "current_attempt_api_call_start_time": marker_a, + "combined_usage_object": Usage(prompt_tokens=4, completion_tokens=1, total_tokens=5), + "response_cost": 0.001, + } + failure_a: Final = httpx.ReadError("attempt A dropped") + coro_a: Final = logging_obj.dispatch_attempt_failure_handlers(failure_a, "") + + logging_obj.carry_failed_attempt_usage() + logging_obj.update_environment_variables( + model="deployment-model-b", + user="", + optional_params={}, + litellm_params=get_litellm_params(metadata={"model_info": {"id": "deployment-b"}}), + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "model": "deployment-model-b", + "custom_llm_provider": "anthropic", + "current_attempt_api_call_start_time": marker_b, + "combined_usage_object": Usage(prompt_tokens=7, completion_tokens=2, total_tokens=9), + "response_cost": 0.002, + } + failure_b: Final = httpx.ReadError("attempt B dropped") + coro_b: Final = logging_obj.dispatch_attempt_failure_handlers(failure_b, "") + + task_a: Final = asyncio.create_task(coro_a) + task_b: Final = asyncio.create_task(coro_b) + try: + await asyncio.wait_for( + asyncio.gather(recorder.attempt_a_started.wait(), recorder.attempt_b_started.wait()), + timeout=10, + ) + request_data: Final = {"litellm_logging_obj": logging_obj} + lifted: Final = _failure_fields_to_lift(request_data, provider_billed=True) + terminal_kwargs: Final = { + key: value + for key, value in {**request_data, **lifted}.items() + if key != "litellm_logging_obj" + } + row: Final = get_logging_payload( + kwargs=terminal_kwargs, + response_obj=failure_b, + start_time=request_start, + end_time=request_start + real_datetime.timedelta(seconds=1), + ) + + assert row["model_id"] == "deployment-b" + assert row["custom_llm_provider"] == "anthropic" + assert row["model"] == "deployment-model-b" + finally: + recorder.release_attempt_a.set() + recorder.release_attempt_b.set() + await asyncio.gather(task_a, task_b) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("queued_start", [False, True], ids=["running-start", "queued-start"]) +async def test_dropped_stream_failure_does_not_overwrite_newer_terminal_payload(queued_start: bool) -> None: + from litellm.exceptions import MidStreamFallbackError + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper + from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload + from litellm.types.utils import Delta, ModelResponseStream, StandardLoggingPayload, StreamingChoices, Usage + from pydantic import TypeAdapter + + request_start: Final = real_datetime.datetime(2026, 1, 1) + model: Final = "deployment-model" + recorder: Final = _ReplaceSharedFailurePayload() + logging_obj: Final = Logging( + model=model, + messages=ONE_USER_MESSAGE, + stream=True, + call_type="acompletion", + start_time=request_start, + litellm_call_id="call-lit145-identity", + function_id="fn-lit145-identity", + dynamic_async_failure_callbacks=[recorder], + ) + recorder.logging_obj = logging_obj + initial_payload: Final = { + "model_id": "older-deployment-id", + "custom_llm_provider": "older-provider", + "model": "older-model", + } + logging_obj.update_environment_variables( + model=model, + user="", + optional_params={}, + litellm_params=get_litellm_params(metadata={"model_info": {"id": "deployment-id"}}), + ) + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "custom_llm_provider": "openai", + "first_api_call_start_time": request_start, + "current_attempt_api_call_start_time": request_start, + "standard_logging_object": initial_payload, + } + + def assert_newer_payload_attribution() -> None: + request_data: Final = {"litellm_logging_obj": logging_obj} + lifted: Final = _failure_fields_to_lift(request_data, provider_billed=True) + terminal_kwargs: Final = { + key: value + for key, value in {**request_data, **lifted}.items() + if key != "litellm_logging_obj" + } + row: Final = get_logging_payload( + kwargs=terminal_kwargs, + response_obj=exception, + start_time=request_start, + end_time=request_start + real_datetime.timedelta(seconds=1), + ) + + assert row["model_id"] == "newer-deployment-id" + assert row["custom_llm_provider"] == "newer-provider" + assert row["model"] == "newer-model" + + exception: Final = httpx.ReadError("connection dropped") + if queued_start: + dispatch: Final = logging_obj.dispatch_attempt_failure_handlers(exception, "") + logging_obj._failure_handler_helper_fn( + exception=httpx.ReadError("newer failure"), + traceback_exception="", + start_time=request_start, + end_time=request_start + real_datetime.timedelta(seconds=1), + ) + replacement_base: Final = logging_obj.model_call_details["standard_logging_object"] + replacement_payload: Final = TypeAdapter(StandardLoggingPayload).validate_python( + { + **replacement_base, + "model_id": "newer-deployment-id", + "custom_llm_provider": "newer-provider", + "model": "newer-model", + } + ) + recorder.pause_until_released = True + recorder.replacement_payload = replacement_payload + logging_obj.model_call_details = { + **logging_obj.model_call_details, + "standard_logging_object": replacement_payload, + } + assert logging_obj.model_call_details["current_attempt_api_call_start_time"] == request_start + + task: Final = asyncio.create_task(dispatch) + try: + await asyncio.wait_for(recorder.started.wait(), timeout=10) + assert_newer_payload_attribution() + finally: + recorder.release.set() + await task + + await asyncio.wait_for(recorder.replaced.wait(), timeout=10) + assert_newer_payload_attribution() + return + + wrapper: Final = CustomStreamWrapper( + completion_stream=None, + model=model, + logging_obj=logging_obj, + custom_llm_provider="openai", + ) + wrapper.chunks = [ + ModelResponseStream( + id="chatcmpl-identity", + created=1742056047, + model=model, + object="chat.completion.chunk", + choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(content="The"))], + usage=Usage(prompt_tokens=30, completion_tokens=1, total_tokens=31), + ) + ] + + with pytest.raises(MidStreamFallbackError): + wrapper._log_stream_failure_and_raise(exception) + await wrapper.wait_for_failure_logging(timeout=10) + await asyncio.wait_for(recorder.replaced.wait(), timeout=10) + assert_newer_payload_attribution() + + def test_a_timed_out_attempt_from_the_group_a_fallback_left_is_still_billed(): """After a fallback hop the router restarts ``attempted_retries`` at 0 for the new model group and re-logs the attempt that exhausted the previous group, so the final attempt (dep-b, retry 1) must be @@ -2655,6 +3336,47 @@ def test_a_request_that_timed_out_at_a_provider_bills_its_input_at_the_input_rat assert response_cost == pytest.approx(expected_input_cost) +@pytest.mark.parametrize( + "current_attempt_api_call_start_time", + [None, 1700000001.0], + ids=["retry-not-dispatched", "retry-dispatched"], +) +def test_a_retry_failure_bills_input_only_for_dispatched_attempts( + current_attempt_api_call_start_time: float | None, +) -> None: + from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift + from litellm.types.utils import Usage + + prior_usage: Final = Usage(prompt_tokens=4, completion_tokens=2, total_tokens=6) + estimated_retry_usage: Final = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None) + assert estimated_retry_usage is not None + + lifted: Final = _failure_fields_to_lift( + { + "litellm_logging_obj": _LoggingObj( + { + "first_api_call_start_time": 1700000000.0, + "current_attempt_api_call_start_time": current_attempt_api_call_start_time, + "failed_attempts_usage": ((prior_usage, 0.25),), + "call_type": "acompletion", + "model": FAILURE_USAGE_MODEL, + "messages": ONE_USER_MESSAGE, + } + ) + } + ) + + usage: Final = lifted["combined_usage_object"] + assert isinstance(usage, Usage) + expected_retry_tokens: Final = ( + estimated_retry_usage.prompt_tokens if current_attempt_api_call_start_time is not None else 0 + ) + assert usage.prompt_tokens == prior_usage.prompt_tokens + expected_retry_tokens + assert usage.completion_tokens == prior_usage.completion_tokens + assert usage.total_tokens == prior_usage.total_tokens + expected_retry_tokens + assert lifted["response_cost"] == 0.25 + + @pytest.mark.parametrize( "model_call_details, dispatched", [ diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index f0f3fc37ddb..e88d951ff09 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -27,7 +27,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.xai.common_utils import xai_reported_cost_in_usd from litellm.proxy._experimental.mcp_server import mcp_server_manager from litellm.proxy._experimental.mcp_server.faults.list_outcomes import ServerListOk -from litellm.proxy.utils import _failure_fields_to_lift +from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.types.llms.openai import ( ResponsesAPIResponse, @@ -1980,7 +1980,22 @@ async def test_non_stream_auto_execute_failing_every_retry_bills_each_paid_first failure_row_fields: Final = _failure_fields_to_lift(MappingProxyType({"litellm_logging_obj": logging_obj})) billed_usage: Final = failure_row_fields["combined_usage_object"] assert isinstance(billed_usage, Usage) - assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == billed_tokens + estimated_failure_usage: Final = ( + _estimate_dispatched_failure_usage( + model="gpt-5.6", + request_input=logging_obj.model_call_details.get("messages"), + system_input=None, + ) + if not retried_first_rounds + else None + ) + expected_prompt_tokens: Final = billed_tokens[0] + ( + estimated_failure_usage.prompt_tokens if estimated_failure_usage is not None else 0 + ) + assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == ( + expected_prompt_tokens, + billed_tokens[1], + ) assert failure_row_fields["response_cost"] == pytest.approx( _round_cost(_ROUND_ONE_BODY) + sum(_round_cost(body) for body in retried_first_rounds) ) diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index 09ce002353f..75b6ab04446 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -2,7 +2,7 @@ import asyncio import importlib import json import time -from collections.abc import AsyncIterator +from collections.abc import AsyncIterator, Mapping from datetime import datetime, timedelta from types import MappingProxyType from typing import Any, Final, Literal, NoReturn @@ -11,6 +11,7 @@ from unittest.mock import MagicMock, patch import httpx import pytest import respx +from pydantic import TypeAdapter import litellm from litellm import Router @@ -19,7 +20,8 @@ from litellm.llms.anthropic.pass_through.messages.fake_stream_iterator import Fa from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure +from litellm.router_utils.cooldown_handlers import async_get_cooldown_deployments, mark_advisor_orchestration_failure +from litellm.types.utils import Choices, Message, ModelResponse, StandardLoggingPayload, Usage from litellm.router_utils.fallback_event_handlers import ( MID_STREAM_FALLBACK_CONTROLS_KEY, PRE_ROUTING_SELECTED_MODEL_KEY, @@ -147,6 +149,220 @@ class RecordingRouter: return StreamingWrapper() +def _request_logging_obj() -> Logging: + return Logging( + model="primary-model", + messages=[{"role": "user", "content": "ping"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime(2026, 1, 1), + litellm_call_id="fallback-hop", + function_id="fallback-hop", + ) + + +class _SuccessRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.logged: tuple[tuple[int, float], ...] = () + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + payload: Final = TypeAdapter(StandardLoggingPayload).validate_python(kwargs["standard_logging_object"]) + self.logged = (*self.logged, (payload["completion_tokens"], payload["response_cost"])) + + +class HopLoggingItsOwnSuccessRouter(FakeRouter): + async def async_function_with_fallbacks(self, *args: object, **kwargs: object) -> StreamingWrapper: + hop_response: Final = ModelResponse( + id="chatcmpl-hop", + model="gpt-5.6", + choices=[Choices(index=0, message=Message(role="assistant", content="served by the hop"))], + usage=Usage(prompt_tokens=52, completion_tokens=7, total_tokens=59), + ) + now: Final = datetime(2026, 1, 1) + logging_obj: Final = kwargs["litellm_logging_obj"] + assert isinstance(logging_obj, Logging) + await logging_obj.async_success_handler(result=hop_response, start_time=now, end_time=now) + return StreamingWrapper() + + +@pytest.mark.asyncio +async def test_run_async_fallback_opens_the_hop_without_the_superseded_attempts_partial_usage() -> None: + """A stream the failed attempt dropped stashed the usage it consumed on the request's logging object. The + hop that replaces it logs its own usage, so the integrations see the hop's tokens and cost, not the stash.""" + recorder: Final = _SuccessRecorder() + logging_obj: Final = Logging( + model="gpt-5.6", + messages=[{"role": "user", "content": "ping"}], + stream=False, + call_type="completion", + start_time=datetime(2026, 1, 1), + litellm_call_id="fallback-hop-success", + function_id="fallback-hop-success", + dynamic_async_success_callbacks=[recorder], + ) + logging_obj.model_call_details["custom_llm_provider"] = "openai" + logging_obj.model_call_details["litellm_params"] = {"acompletion": True} + logging_obj.model_call_details["current_attempt_api_call_start_time"] = datetime(2026, 1, 1) + logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] + partial_usage: Final = Usage(prompt_tokens=52, completion_tokens=1, total_tokens=53) + logging_obj.record_partial_usage_for_failure(partial_usage, 0.01) + + await run_async_fallback( + litellm_router=HopLoggingItsOwnSuccessRouter(), + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("stream dropped before content"), + max_fallbacks=3, + fallback_depth=0, + litellm_logging_obj=logging_obj, + ) + + assert tuple(tokens for tokens, _ in recorder.logged) == (7,) + assert recorder.logged[0][1] != 0.01 + assert logging_obj.model_call_details["failed_attempts_usage"] == ((partial_usage, 0.01),) + assert logging_obj.model_call_details["current_attempt_api_call_start_time"] is None + + +@pytest.mark.asyncio +async def test_run_async_fallback_preserves_superseded_usage_when_hop_fails_before_opening() -> None: + logging_obj: Final = _request_logging_obj() + partial_usage: Final = Usage(prompt_tokens=52, completion_tokens=1, total_tokens=53) + logging_obj.record_partial_usage_for_failure(partial_usage, 0.01) + + with pytest.raises(RuntimeError, match="fallback model also failed"): + await run_async_fallback( + litellm_router=AlwaysFailRouter(), + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("stream dropped before content"), + max_fallbacks=3, + fallback_depth=0, + litellm_logging_obj=logging_obj, + ) + + assert logging_obj.model_call_details["failed_attempts_usage"] == ((partial_usage, 0.01),) + + +class _RateLimitedHop(Exception): + def __init__(self, deployment_id: str = "fallback-dep-1") -> None: + super().__init__("fallback deployment rate limited") + self.status_code = 429 + self.failed_deployment_id = deployment_id + + +_EARLIER_FAILURE_MARKER: Final = datetime(2026, 1, 1) +_FIRST_HOP_FAILURE_MARKER: Final = datetime(2026, 1, 2) +_FINAL_HOP_FAILURE_MARKER: Final = datetime(2026, 1, 3) + + +class _TwoFailureFallbackRouter(Router): + def __init__(self) -> None: + super().__init__( + model_list=[ + { + "model_name": "fallback-model", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + "model_info": {"id": f"fallback-dep-{index}"}, + } + for index in (1, 2) + ], + allowed_fails=0, + cooldown_time=60, + num_retries=0, + ) + + async def async_function_with_fallbacks(self, *args: object, **kwargs: object) -> NoReturn: + logging_obj: Final[object] = kwargs["litellm_logging_obj"] + assert isinstance(logging_obj, Logging) + assert logging_obj.model_call_details.get("has_logged_async_failure", False) is False + + logging_obj.model_call_details["current_attempt_api_call_start_time"] = _FIRST_HOP_FAILURE_MARKER + try: + raise _RateLimitedHop("fallback-dep-1") + except _RateLimitedHop: + logging_obj.mark_logging_complete("async_failure") + + logging_obj.model_call_details["current_attempt_api_call_start_time"] = _FINAL_HOP_FAILURE_MARKER + raise _RateLimitedHop("fallback-dep-2") + + +def _two_deployment_router() -> Router: + return Router( + model_list=[ + { + "model_name": "fallback-model", + "litellm_params": {"model": "openai/gpt-5.6", "api_key": "fake-key"}, + "model_info": {"id": f"fallback-dep-{index}"}, + } + for index in (1, 2) + ], + num_retries=0, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure_already_logged", [False, True]) +async def test_run_async_fallback_cools_down_a_failed_hop_once(failure_already_logged: bool) -> None: + """The manual cooldown stands in for the failure callbacks the logging object blocks once a failure has + been logged. A hop whose own failure was the first one logged already ran them, so only a hop that failed + with the flag already set gets the manual cooldown.""" + router: Final = _two_deployment_router() + logging_obj: Final = _request_logging_obj() + if failure_already_logged: + logging_obj.model_call_details["current_attempt_api_call_start_time"] = _EARLIER_FAILURE_MARKER + logging_obj.mark_logging_complete("async_failure") + + async def rate_limited_hop(*args: object, **kwargs: object) -> NoReturn: + hop_logging_obj: Final = kwargs["litellm_logging_obj"] + assert isinstance(hop_logging_obj, Logging) + hop_logging_obj.model_call_details["current_attempt_api_call_start_time"] = _FINAL_HOP_FAILURE_MARKER + if hop_logging_obj.model_call_details.get("has_logged_async_failure", False) is not True: + hop_logging_obj.mark_logging_complete("async_failure") + raise _RateLimitedHop() + + with pytest.raises(_RateLimitedHop): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("stream dropped before content"), + max_fallbacks=3, + fallback_depth=0, + litellm_logging_obj=logging_obj, + original_function=rate_limited_hop, + metadata={}, + ) + + cooled_down: Final = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) + assert cooled_down == (["fallback-dep-1"] if failure_already_logged else []) + + +@pytest.mark.asyncio +async def test_run_async_fallback_cools_down_the_final_deployment_after_retry_failure_logging() -> None: + router: Final = _TwoFailureFallbackRouter() + logging_obj: Final = _request_logging_obj() + assert logging_obj.model_call_details.get("has_logged_async_failure", False) is False + + with pytest.raises(_RateLimitedHop): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["fallback-model"], + original_model_group="primary-model", + original_exception=RuntimeError("stream dropped before content"), + max_fallbacks=3, + fallback_depth=0, + litellm_logging_obj=logging_obj, + metadata={}, + ) + + assert logging_obj.model_call_details.get("has_logged_async_failure", False) is True + cooled_down: Final = await async_get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) + assert cooled_down == ["fallback-dep-2"] + + @pytest.mark.asyncio async def test_run_async_fallback_forwards_include_fallback_errors_to_nested_call(): """A nested fallback (multi-hop) must keep collecting errors, so the opt-in @@ -1163,11 +1379,18 @@ class TestRunAsyncFallbackTriggersCooldown: return None async def async_function_with_fallbacks(self, *args, **kwargs): + logging_obj: Final = kwargs.get("litellm_logging_obj") + if isinstance(logging_obj, Logging): + logging_obj.model_call_details["current_attempt_api_call_start_time"] = _FINAL_HOP_FAILURE_MARKER + if logging_obj.model_call_details.get("has_logged_async_failure", False) is not True: + logging_obj.mark_logging_complete("async_failure") raise RuntimeError("fallback model also failed") - def _logging_obj(self, has_logged_async_failure: bool) -> MagicMock: - logging_obj = MagicMock() - logging_obj.model_call_details = {"has_logged_async_failure": has_logged_async_failure} + def _logging_obj(self, has_logged_async_failure: bool) -> Logging: + logging_obj: Final = _request_logging_obj() + if has_logged_async_failure: + logging_obj.model_call_details["current_attempt_api_call_start_time"] = _EARLIER_FAILURE_MARKER + logging_obj.mark_logging_complete("async_failure") return logging_obj @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index b119d216542..e1f748fcb58 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -23,7 +23,7 @@ import pytest import respx from fastapi import HTTPException from opentelemetry import trace -from pydantic import BaseModel +from pydantic import BaseModel, TypeAdapter import litellm from litellm import APIConnectionError, Router @@ -93,7 +93,7 @@ from litellm.llms.base_llm.vector_store.transformation import( LiteLLMVectorStoreEmbeddingExecutor, RouterVectorStoreEmbeddingExecutor, ) -from litellm.types.utils import CallTypes, CredentialItem +from litellm.types.utils import CallTypes, CredentialItem, Usage from litellm.utils import _invalidate_model_cost_lowercase_map from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome from tests.fake_openai_endpoint import FAKE_OPENAI_API_BASE @@ -16726,6 +16726,7 @@ class _AnthropicMessagesFakeByteStream: self._hidden_params = {"additional_headers": {"x-amzn-requestid": "req-1"}} self.closed = False self.closed_as_superseded = False + self.close_contexts: tuple[bool, ...] = () def __aiter__(self): return self @@ -16736,6 +16737,7 @@ class _AnthropicMessagesFakeByteStream: return self._chunks.pop(0) async def aclose(self) -> None: + self.close_contexts = (*self.close_contexts, closing_superseded_attempt()) self.closed_as_superseded = self.closed_as_superseded or closing_superseded_attempt() self.closed = True @@ -17979,6 +17981,51 @@ async def test_anthropic_messages_stream_dropped_before_content_is_retried_withi assert [(attempted, budget) for _, attempted, budget in provider.calls] == [(0, 2), (1, 2)] +class _AnthropicMessagesRetryStashRecorder: + def __init__(self) -> None: + self.observed: tuple[Usage | None, ...] = () + + def record(self, usage: Usage | None) -> None: + self.observed = (*self.observed, usage) + + +@pytest.mark.asyncio +async def test_anthropic_messages_retry_opens_without_the_superseded_attempts_partial_usage() -> None: + """The dropped attempt's usage stays available for terminal billing without reaching the retry's success log.""" + router: Final = _anthropic_messages_retry_router(num_retries=1) + logging_obj: Final = LiteLLMLogging( + model="glm", + messages=[{"role": "user", "content": "ping"}], + stream=True, + call_type="anthropic_messages", + start_time=datetime(2026, 1, 1), + litellm_call_id="retry-own-usage", + function_id="retry-own-usage", + ) + partial_usage: Final = Usage(prompt_tokens=52, completion_tokens=1, total_tokens=53) + logging_obj.record_partial_usage_for_failure(partial_usage, 0.01) + stash_recorder: Final = _AnthropicMessagesRetryStashRecorder() + usage_adapter: Final = TypeAdapter(Usage | None) + + def retry_stream() -> _AnthropicMessagesFakeByteStream: + stash_recorder.record( + usage_adapter.validate_python(logging_obj.model_call_details.get("combined_usage_object")) + ) + return _anthropic_messages_retried_stream() + + provider: Final = _AnthropicMessagesScriptedProvider(_anthropic_messages_dropped_before_content, retry_stream) + + stream: Final = await _anthropic_messages_stream_through_router( + router, provider, litellm_logging_obj=logging_obj + ) + body: Final = tuple([chunk async for chunk in stream]) + + assert stash_recorder.observed == (None,) + assert body == (_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")) + assert "response_cost" not in logging_obj.model_call_details + assert logging_obj.model_call_details["failed_attempts_usage"] == ((partial_usage, 0.01),) + + @pytest.mark.asyncio async def test_anthropic_messages_superseded_stream_is_closed_before_the_retry_opens(): """Both attempts log through the request's one logging object, so the superseded attempt's end-of-stream @@ -18001,6 +18048,7 @@ async def test_anthropic_messages_superseded_stream_is_closed_before_the_retry_o assert body == [_anthropic_messages_message_start_chunk(), _anthropic_messages_content_chunk("pong")] assert superseded_closed_when_retry_opened == [True] + assert superseded.close_contexts[0] is True @pytest.mark.asyncio From 159b5b2f7243bba757c08e4ddc45a2514e4cd037 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Tue, 20 Oct 2026 21:52:01 -0700 Subject: [PATCH 054/235] chore(cost-map): add fireworks muse spark open priority prices from the pricing page (#47785) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 6 +++++- model_prices_and_context_window.json | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 764cf397158..cb9bfa6ffe4 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -64195,13 +64195,17 @@ }, "fireworks_ai/accounts/fireworks/models/muse-spark-open": { "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 1.875e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, - "source": "https://fireworks.ai/models/fireworks/muse-spark-open", + "output_cost_per_token_priority": 6.375e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, + "supports_reasoning": true, "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/nemotron-lightning-3p5-30b-a3b": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 764cf397158..cb9bfa6ffe4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -64195,13 +64195,17 @@ }, "fireworks_ai/accounts/fireworks/models/muse-spark-open": { "cache_read_input_token_cost": 1.5e-07, + "cache_read_input_token_cost_priority": 2.25e-07, "input_cost_per_token": 1.25e-06, + "input_cost_per_token_priority": 1.875e-06, "litellm_provider": "fireworks_ai", "max_input_tokens": 1048576, "mode": "chat", "output_cost_per_token": 4.25e-06, - "source": "https://fireworks.ai/models/fireworks/muse-spark-open", + "output_cost_per_token_priority": 6.375e-06, + "source": "https://docs.fireworks.ai/serverless/pricing", "supports_function_calling": true, + "supports_reasoning": true, "supports_vision": true }, "fireworks_ai/accounts/fireworks/models/nemotron-lightning-3p5-30b-a3b": { From b99d2cc06e9f5a05765228b85bc64d2a090ffd00 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 22:31:08 -0700 Subject: [PATCH 055/235] test(perf): ramp the load cases until the pod stops keeping up and floor sustained_rps (#47758) * test(perf): add small_bodies_30rps and mixed_size_requests_4rps load cases Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(perf): run the load cases at 50 rps small and 5 rps mixed, the pod's knee Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(perf): ramp the load cases until the pod stops keeping up and report sustained_rps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(perf): hand the ramp to the observation and widen the mixed ramp to 14 rps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(perf): floor the small ramp at 20 rps and emit the per-step readings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(perf): emit the ramp's hold bound and each step's unexpected statuses Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(perf): read each ramp step's cpu, redis latency and connections from the samplers' windows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(perf): floor mixed_size_requests_ramp at 8 rps from the lane's 10 rps sustained step Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(perf): compare step readings with math.isclose so the e2e basedpyright gate stays clean Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/e2e/AGENTS.md | 2 +- .../perf/cases/mixed_size_requests_ramp.py | 24 +++ tests/e2e/perf/cases/small_bodies_ramp.py | 24 +++ tests/e2e/perf/setup/datastore.py | 18 +- tests/e2e/perf/setup/locustfile.py | 15 +- tests/e2e/perf/setup/observe.py | 129 ++++++++++++-- tests/e2e/perf/setup/performance_test_case.py | 78 +++++++- tests/e2e/perf/setup/plan.py | 4 + tests/e2e/perf/setup/podcpu.py | 18 +- tests/e2e/perf/setup/ramp.py | 116 ++++++++++++ tests/e2e/perf/setup/run.py | 1 + tests/e2e/perf/setup/series.py | 48 +++++ tests/e2e/perf/setup/telemetry.py | 51 +++++- tests/e2e/perf/setup/upgrade.py | 4 +- tests/e2e/perf/setup/wire.py | 2 + tests/e2e/perf/test_perf_cases.py | 15 +- tests/e2e_harness/perf/setup/test_ramp.py | 168 ++++++++++++++++++ tests/e2e_harness/perf/test_upgrade.py | 4 +- 18 files changed, 677 insertions(+), 44 deletions(-) create mode 100644 tests/e2e/perf/cases/mixed_size_requests_ramp.py create mode 100644 tests/e2e/perf/cases/small_bodies_ramp.py create mode 100644 tests/e2e/perf/setup/ramp.py create mode 100644 tests/e2e/perf/setup/series.py create mode 100644 tests/e2e_harness/perf/setup/test_ramp.py diff --git a/tests/e2e/AGENTS.md b/tests/e2e/AGENTS.md index d976a2b5ee3..ef57135a4d2 100644 --- a/tests/e2e/AGENTS.md +++ b/tests/e2e/AGENTS.md @@ -44,7 +44,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family - `logging/` - logging-integration delivery (datadog and friends) - `security/` - secret handling and log-leak protection - `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did, and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a loose ceiling over the measured batches nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak) -- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in. The perf cases (`test_perf_cases.py`, one `PerformanceTestCase` per module in `perf/cases/`, no `_e2e` suffix) are the request-performance runs. A case names what the gateway was deployed with (model groups and guardrails, written out in the gateway's config: the `proxy_config` block of project-releaser's perf helm values in the lane, `gateway/perf_ci_config.yml` locally, with each deployment's mock knobs in an `x-mock-knobs` extra header so the model ids are real ones like `gpt-6-luna` and `anthropic.claude-sonnet-5`) and lists what the test creates and sends: teams, Keycloak JWT users, virtual keys, `RequestShape`s (a coding-agent session from `perf/setup/corpus/` whose input-token size is drawn per request from the shape's range, with or without tools, and the output cap), a `Traffic` plan (seed, rps, request count, endpoint, shape and stream shares) and `Limit`s (one metric at one percentile under one ceiling). The infra side lives in `perf/setup/`: `provision.py` checks the deployed config matches the case and builds the callers, `plan.py` draws the request list from the seed, `run.py` replays it from a separate Locust process (`locustfile.py`) that builds each body right before sending and records status, header, first-byte, first-token and total time per request, and `observe.py` joins all of that with what the mocks received and the workers' RSS to score every `Limit`. Limits are about what the caller sees and what the gateway costs: every non-blocked request is a 200 (`unexpected_status_count`), the seconds the gateway added per request at p50 and p99 (`gateway_added_seconds`, to the first token on streams), the client's time to the first token of a stream at p50 and p99 (`time_to_first_token_seconds`, mock delay included), worker memory growth (`memory_growth_megabytes`), and the Postgres statements and Redis commands a request waits on before it is forwarded (`postgres_calls_per_request`, `redis_calls_per_request`: auth and routing reads, the rate-limiter script), with the work the gateway does after the response or on a timer (spend writes, daily aggregates, config reloads) reported separately as `postgres_background_calls_per_request` and `redis_background_calls_per_request`; the same overhead as the gateway itself reports it, read from the `x-litellm-overhead-duration-ms` header per request (`litellm_overhead_seconds`, gated next to `gateway_added_seconds`, the outside reading of the same thing, so the two can be charted together) and the gateway's own end-to-end time from `x-litellm-response-duration-ms` (`litellm_response_seconds`), the run mean of the gateway's `litellm_redis_latency` histogram from a `/metrics` scrape before and after the run (`telemetry.py`, `prometheus_redis_latency_mean_seconds`), and the database's own view of the run from the same `pg_stat_statements` rows and a once-a-second `pg_stat_activity` sample: execution seconds per wall second (`postgres_active_sessions`, the average-active-sessions number RDS reports as DB load), the slowest statement's mean (`postgres_slowest_statement_seconds`, named in the detail) and the peak client connection count (`postgres_peak_connections`), each under a ceiling set from the first local runs and due a retune once the lane has its ten-run baseline; `datastore.py` reads `pg_stat_statements` before and after the run and holds a Redis `MONITOR` connection during it, sorts each statement by verb and table and each command by key, and the run output lists the top statements and commands per bucket so a red run names what grew. Whether a body, a guardrail call or a spend row is right is printed as a diagnostic, never a limit, that is the functional suites' job. The providers and the guardrail are out-of-process servers in `perf/setup/mocks/` (`server.py openai|anthropic|bedrock|guardrail`, each answering with a reply generated for the request in that provider's wire format, sized from the prompt and output cap, a tool call when tools were offered, waiting the case's `mock_latency` first, request logs at `/mock/requests` and `/mock/calls`), reached through `E2E_PERF_MOCK_*_URL` by both the gateway config and the test. Adding a perf test is adding a module with a `CASE` to `perf/cases/`; a new deployment or guardrail is a config change; a new metric is a name in `Metric` plus a branch in `observe.observe` +- `perf/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163). Do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend, marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see), needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_PERF_TESTS` is set and excluded from the per-PR selector like the rest of `perf/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in). New perf tests go here too, marked `perf`, behind the same `E2E_PERF_TESTS` opt-in. The perf cases (`test_perf_cases.py`, one `PerformanceTestCase` per module in `perf/cases/`, no `_e2e` suffix) are the request-performance runs. A case names what the gateway was deployed with (model groups and guardrails, written out in the gateway's config: the `proxy_config` block of project-releaser's perf helm values in the lane, `gateway/perf_ci_config.yml` locally, with each deployment's mock knobs in an `x-mock-knobs` extra header so the model ids are real ones like `gpt-6-luna` and `anthropic.claude-sonnet-5`) and lists what the test creates and sends: teams, Keycloak JWT users, virtual keys, `RequestShape`s (a coding-agent session from `perf/setup/corpus/` whose input-token size is drawn per request from the shape's range, with or without tools, and the output cap), a `Traffic` plan (seed, rps, request count, endpoint, shape and stream shares, or a `Ramp` that steps the rate up through the run, built with `ramped()`, whose plan carries each request's send second and whose run reports `sustained_rps`, the last step the pod held, scored in `perf/setup/ramp.py`, with every step's latency, completed rate, non-200s, CPU per request, Redis latency and Postgres connections cut from the samplers' time-stamped `Series` (`perf/setup/series.py`) to the step's window and emitted as `perf_ramp_step__` JUnit properties for the dashboard) and `Limit`s (one metric at one percentile under one ceiling, or over one floor for `sustained_rps`). The infra side lives in `perf/setup/`: `provision.py` checks the deployed config matches the case and builds the callers, `plan.py` draws the request list from the seed, `run.py` replays it from a separate Locust process (`locustfile.py`) that builds each body right before sending and records status, header, first-byte, first-token and total time per request, and `observe.py` joins all of that with what the mocks received and the workers' RSS to score every `Limit`. Limits are about what the caller sees and what the gateway costs: every non-blocked request is a 200 (`unexpected_status_count`), the seconds the gateway added per request at p50 and p99 (`gateway_added_seconds`, to the first token on streams), the client's time to the first token of a stream at p50 and p99 (`time_to_first_token_seconds`, mock delay included), worker memory growth (`memory_growth_megabytes`), and the Postgres statements and Redis commands a request waits on before it is forwarded (`postgres_calls_per_request`, `redis_calls_per_request`: auth and routing reads, the rate-limiter script), with the work the gateway does after the response or on a timer (spend writes, daily aggregates, config reloads) reported separately as `postgres_background_calls_per_request` and `redis_background_calls_per_request`; the same overhead as the gateway itself reports it, read from the `x-litellm-overhead-duration-ms` header per request (`litellm_overhead_seconds`, gated next to `gateway_added_seconds`, the outside reading of the same thing, so the two can be charted together) and the gateway's own end-to-end time from `x-litellm-response-duration-ms` (`litellm_response_seconds`), the run mean of the gateway's `litellm_redis_latency` histogram from a `/metrics` scrape before and after the run (`telemetry.py`, `prometheus_redis_latency_mean_seconds`), and the database's own view of the run from the same `pg_stat_statements` rows and a once-a-second `pg_stat_activity` sample: execution seconds per wall second (`postgres_active_sessions`, the average-active-sessions number RDS reports as DB load), the slowest statement's mean (`postgres_slowest_statement_seconds`, named in the detail) and the peak client connection count (`postgres_peak_connections`), each under a ceiling set from the first local runs and due a retune once the lane has its ten-run baseline; `datastore.py` reads `pg_stat_statements` before and after the run and holds a Redis `MONITOR` connection during it, sorts each statement by verb and table and each command by key, and the run output lists the top statements and commands per bucket so a red run names what grew. Whether a body, a guardrail call or a spend row is right is printed as a diagnostic, never a limit, that is the functional suites' job. The providers and the guardrail are out-of-process servers in `perf/setup/mocks/` (`server.py openai|anthropic|bedrock|guardrail`, each answering with a reply generated for the request in that provider's wire format, sized from the prompt and output cap, a tool call when tools were offered, waiting the case's `mock_latency` first, request logs at `/mock/requests` and `/mock/calls`), reached through `E2E_PERF_MOCK_*_URL` by both the gateway config and the test. Adding a perf test is adding a module with a `CASE` to `perf/cases/`; a new deployment or guardrail is a config change; a new metric is a name in `Metric` plus a branch in `observe.observe` - `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at, see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite - `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager__ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up ` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only) - `gateway/` - proxy configuration only (`litellm-config.yml`). No tests diff --git a/tests/e2e/perf/cases/mixed_size_requests_ramp.py b/tests/e2e/perf/cases/mixed_size_requests_ramp.py new file mode 100644 index 00000000000..d7db545c031 --- /dev/null +++ b/tests/e2e/perf/cases/mixed_size_requests_ramp.py @@ -0,0 +1,24 @@ +"""The mixed-size traffic stepped up until the pod stops keeping up: the same callers, guardrails, mocks and body +mix (short chats, 300k to 1M-token contexts, tool calls, half streamed) as `mixed_size_requests_2rps`, 2 rps more +every 90 s from 2 to 14 on the same one 4-worker pod. Where the 2 rps case prices one big body on its own, this one +reports `sustained_rps`, the last step the pod held with big bodies overlapping in every worker, so a handler that +re-walks a 1M-token body on the event loop or a worker that holds several of them in memory at once moves the knee +down and fails the floor.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Final + +from setup.performance_test_case import Limit, Ramp, ramped + +from cases.mixed_size_requests_2rps import CASE as MIXED + +RAMP: Final = Ramp(first_rps=2.0, last_rps=14.0, step_rps=2.0, step_seconds=90.0, held_p99_seconds=5.0) + +CASE: Final = replace( + MIXED, + name="mixed_size_requests_ramp", + traffic=ramped(MIXED.traffic, RAMP, seed=9, concurrency=300), + limits=(Limit("sustained_rps", min=8),), +) diff --git a/tests/e2e/perf/cases/small_bodies_ramp.py b/tests/e2e/perf/cases/small_bodies_ramp.py new file mode 100644 index 00000000000..64308168cee --- /dev/null +++ b/tests/e2e/perf/cases/small_bodies_ramp.py @@ -0,0 +1,24 @@ +"""The small-bodies traffic stepped up until the pod stops keeping up: the same callers, guardrails, mocks and +2.5k to 4k-token bodies as `small_bodies_5rps`, 10 rps more every 90 s from 10 to 70 on the same one 4-worker pod. +The 5 rps case prices one request with the pod idle; this one reports `sustained_rps`, the last step the pod +held (gateway added p99 under 5 s, the queue still draining), so a change that moves the knee down, a lock held +across an await, a pool that fills, a flush that stalls the loop while dozens of requests are in flight, fails the +floor instead of hiding under a fixed rate the pod still manages.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Final + +from setup.performance_test_case import Limit, Ramp, ramped + +from cases.small_bodies_5rps import CASE as SMALL + +RAMP: Final = Ramp(first_rps=10.0, last_rps=70.0, step_rps=10.0, step_seconds=90.0, held_p99_seconds=5.0) + +CASE: Final = replace( + SMALL, + name="small_bodies_ramp", + traffic=ramped(SMALL.traffic, RAMP, seed=13, concurrency=400), + limits=(Limit("sustained_rps", min=20),), +) diff --git a/tests/e2e/perf/setup/datastore.py b/tests/e2e/perf/setup/datastore.py index 89739b6d13e..6ce37da3554 100644 --- a/tests/e2e/perf/setup/datastore.py +++ b/tests/e2e/perf/setup/datastore.py @@ -22,6 +22,8 @@ from typing import TYPE_CHECKING, Final, Literal from pydantic import TypeAdapter +from setup.series import Recorder, Series + if TYPE_CHECKING: import redis @@ -182,13 +184,22 @@ def slowest_statement(statements: tuple[StatementCalls, ...] | None) -> Statemen class ConnectionsSampler: """`pg_stat_activity` client backends on the gateway's database once a second on a background thread, minus - this sampler's own connection, so the peak is what the gateway (and the test's own reads) held open.""" + this sampler's own connection, so the peak is what the gateway (and the test's own reads) held open. + `series` keeps every count with its time for a ramped run's per-step peaks.""" def __init__(self, url: str) -> None: self._url: Final = url self._stop: Final = threading.Event() self._thread: Final = threading.Thread(target=self._run, name="postgres-connections", daemon=True) - self.peak: int | None = None + self._counts: Final[Recorder[int]] = Recorder() + + @property + def peak(self) -> int | None: + return max((sample.value for sample in self._counts.series.samples), default=None) + + @property + def series(self) -> Series[int]: + return self._counts.series def start(self) -> None: self._thread.start() @@ -203,12 +214,13 @@ class ConnectionsSampler: with psycopg.connect(self._url, autocommit=True) as conn: while not self._stop.is_set(): (count,) = _COMMITS_ROW.validate_python(conn.execute(_CONNECTIONS_QUERY).fetchone()) - self.peak = count if self.peak is None else max(self.peak, count) + self._counts.record(count) _ = self._stop.wait(_CONNECTIONS_SAMPLE_SECONDS) class NoConnectionsSampler: peak: None = None + series: None = None def start(self) -> None: return None diff --git a/tests/e2e/perf/setup/locustfile.py b/tests/e2e/perf/setup/locustfile.py index 1c38eae22a1..82a5749becf 100644 --- a/tests/e2e/perf/setup/locustfile.py +++ b/tests/e2e/perf/setup/locustfile.py @@ -4,7 +4,8 @@ time to headers, time to first byte, time to first token (streams), total time, and whether it matched. Runs in its own process (`setup/run.py` starts it), never in the gateway's. Reads `PERF_PLAN_FILE` (JSON list of wire requests), writes `PERF_RECORDS_FILE` (one JSON object per -line), paces to `PERF_RPS` over `PERF_USERS` users, and quits once the last request is recorded. +line), paces to `PERF_RPS` over `PERF_USERS` users (or, when the plan carries `send_at_s`, sends each +request at its scheduled second), and quits once the last request is recorded. """ from __future__ import annotations @@ -35,6 +36,8 @@ _JITTER: Final = 0.25 _RNG: Final = random.Random(_PLAN[0].marker if _PLAN else "empty-plan") _RECORDS: Final = _RECORDS_PATH.open("a", encoding="utf-8") _PROGRESS: Final = PlanProgress(len(_PLAN)) +_SCHEDULED: Final = bool(_PLAN) and _PLAN[0].send_at_s is not None +_RUN_STARTED: Final = time.monotonic() _STOP_FILE: Final = os.environ.get("PERF_STOP_FILE") BLOCK_REASON: Final = "perf-test-block" _TOKEN_MAX_AGE_S: Final = 240.0 @@ -211,7 +214,8 @@ def _authorization(user: FastHttpUser, item: WireRequest) -> str: class PlanUser(FastHttpUser): """One simulated client; all users share the plan and together hold the planned rate. Each user starts at a random point of its first interval and jitters every later interval by up to `_JITTER` either way, so the - users never fire in lockstep; the mean interval is unchanged and `rps` stays the total rate.""" + users never fire in lockstep; the mean interval is unchanged and `rps` stays the total rate. A scheduled + plan (a ramp) is paced by the plan instead: a user takes the next request and sleeps until its second.""" network_timeout = float(os.environ.get("PERF_REQUEST_TIMEOUT", "120")) connection_timeout = 30.0 @@ -219,9 +223,12 @@ class PlanUser(FastHttpUser): def on_start(self) -> None: self._last_run = 0.0 self._last_wait = 0.0 - gevent.sleep(_RNG.uniform(0.0, _INTERVAL_S)) + if not _SCHEDULED: + gevent.sleep(_RNG.uniform(0.0, _INTERVAL_S)) def wait_time(self) -> float: + if _SCHEDULED: + return 0.1 target: Final = _INTERVAL_S * _RNG.uniform(1.0 - _JITTER, 1.0 + _JITTER) now: Final = time.monotonic() run_s: Final = now - self._last_run - self._last_wait if self._last_run else 0.0 @@ -237,6 +244,8 @@ class PlanUser(FastHttpUser): if _PROGRESS.drained: _quit(self) return + if item.send_at_s is not None: + gevent.sleep(max(0.0, _RUN_STARTED + item.send_at_s - time.monotonic())) started_at: Final = time.time() started: Final = time.perf_counter() first_byte_s: float | None = None diff --git a/tests/e2e/perf/setup/observe.py b/tests/e2e/perf/setup/observe.py index 1532a78606e..dccc6e272b3 100644 --- a/tests/e2e/perf/setup/observe.py +++ b/tests/e2e/perf/setup/observe.py @@ -7,14 +7,16 @@ import hashlib import json import statistics from collections import Counter, defaultdict -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from dataclasses import asdict, dataclass -from itertools import chain +from itertools import chain, groupby +from types import MappingProxyType from typing import Final, Literal from e2e_http import NoBody, get_external, post_external, unwrap from memory_readings import RssCapture, RssReading, WorkerKey from pydantic import BaseModel +from typing_extensions import ReadOnly, TypedDict from setup.datastore import ( DbCounters, @@ -27,7 +29,9 @@ from setup.datastore import ( statement_deltas, ) from setup.mocks.reply_common import Provider -from setup.performance_test_case import Limit, Metric, MockLatency, PerformanceTestCase +from setup.performance_test_case import Limit, Metric, MockLatency, PerformanceTestCase, Ramp +from setup.ramp import StepReading, render_steps, sustained_rps +from setup.series import Series from setup.telemetry import GatewayTelemetry from setup.wire import WireRecord @@ -122,6 +126,10 @@ class Observation: telemetry_before: GatewayTelemetry | None = None telemetry_after: GatewayTelemetry | None = None peak_connections: int | None = None + ramp: Ramp | None = None + cpu_series: Series[float] | None = None + connection_series: Series[int] | None = None + telemetry_series: Series[GatewayTelemetry] | None = None @property def telemetry(self) -> GatewayTelemetry | None: @@ -239,6 +247,67 @@ class Observation: post: Final = Counter(call.marker for call in self.guardrail_calls if call.input_type == "response") return sum(1 for record in self.served if pre[record.marker] != 1 or post[record.marker] != 1) + def steps(self) -> tuple[StepReading, ...]: + """One reading per step of a ramped run: the gateway's added latency over the step's requests, the rate + at which requests completed inside the step's window, the step's unexpected statuses, and what the pod + was doing in that window (CPU per completed request, Redis latency, Postgres connections).""" + ramp: Final = self.ramp + if ramp is None or not self.records: + return () + started: Final = min(record.started_at for record in self.records) + added: Final = _by_step(ramp, self.latency_pairs("gateway_added_seconds")) + first_token: Final = _by_step(ramp, self.latency_pairs("time_to_first_token_seconds")) + completed: Final = Counter( + int((record.started_at + record.total_s - started) // ramp.step_seconds) for record in self.records + ) + unexpected: Final = Counter( + ramp.step_of(record.seq) for record in self.records if not record.blocked and record.status != 200 + ) + return tuple( + StepReading( + rps=rate, + planned=count, + completed_rps=completed[index] / ramp.step_seconds, + gateway_added_p99=percentile(added.get(index, ()), 99), + unexpected=unexpected[index], + held_p99_seconds=ramp.held_p99_seconds, + gateway_added_p50=percentile(added.get(index, ()), 50), + time_to_first_token_p99=percentile(first_token.get(index, ()), 99), + **self._window_readings(started + index * ramp.step_seconds, ramp.step_seconds, completed[index]), + ) + for index, (rate, count) in enumerate(zip(ramp.rates, ramp.counts, strict=True)) + ) + + def _window_readings(self, start: float, seconds: float, completed: int) -> WindowReadings: + end: Final = start + seconds + cpu: Final = None if self.cpu_series is None else sum(self.cpu_series.between(start, end)) + conns: Final = None if self.connection_series is None else self.connection_series.between(start, end) + before: Final = None if self.telemetry_series is None else self.telemetry_series.last_before(start) + after: Final = None if self.telemetry_series is None else self.telemetry_series.last_before(end) + return WindowReadings( + cpu_seconds_per_request=None if cpu is None or completed == 0 else cpu / completed, + redis_latency_mean_seconds=None + if before is None or after is None + else after.minus(before).redis_latency.mean, + postgres_peak_connections=max(conns, default=None) if conns else None, + ) + + +class WindowReadings(TypedDict): + cpu_seconds_per_request: ReadOnly[float | None] + redis_latency_mean_seconds: ReadOnly[float | None] + postgres_peak_connections: ReadOnly[int | None] + + +def _by_step(ramp: Ramp, pairs: Iterable[tuple[WireRecord, float]]) -> Mapping[int, tuple[float, ...]]: + ordered: Final = sorted(pairs, key=lambda pair: pair[0].seq) + return MappingProxyType( + { + step: tuple(value for _, value in group) + for step, group in groupby(ordered, key=lambda pair: ramp.step_of(pair[0].seq)) + } + ) + def _text_sha1(text: str, *, intervened: bool) -> str: original: Final = text.removesuffix(" [post]") if intervened else text @@ -415,6 +484,8 @@ def observe( if metric == "cpu_seconds_per_request": cpu: Final = observation.gateway_cpu_seconds return None if cpu is None or not observation.records else cpu / len(observation.records) + if metric == "sustained_rps": + return None if observation.ramp is None else sustained_rps(observation.steps()) return worst_worker_growth_mb(observation.rss_before, observation.rss_after) @@ -424,6 +495,7 @@ class Row: percentile: int | None observed: float | None limit: float | None + floor: float | None = None detail: str = "" breakdown: tuple[Contributor, ...] = () @@ -437,11 +509,22 @@ class Row: @property def over(self) -> bool: - return self.limit is not None and self.observed is not None and self.observed > self.limit + """Past the limit: above a max, or below a floor.""" + if self.observed is None: + return False + return (self.limit is not None and self.observed > self.limit) or ( + self.floor is not None and self.observed < self.floor + ) @property def missing(self) -> bool: - return self.limit is not None and self.observed is None + return (self.limit is not None or self.floor is not None) and self.observed is None + + @property + def bound(self) -> str: + if self.limit is not None: + return f"{self.limit:.4f}" + return "" if self.floor is None else f">= {self.floor:.4f}" REPORTED: Final[tuple[tuple[Metric, int | None], ...]] = ( @@ -476,6 +559,8 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[ (limit.metric, limit.percentile): limit for limit in case.limits } keys: list[tuple[Metric, int | None]] = list(REPORTED) + if observation.ramp is not None: + keys.append(("sustained_rps", None)) keys.extend(key for key in limits if key not in keys) rows: list[Row] = [] for metric, rank in keys: @@ -486,20 +571,23 @@ def evaluate(case: PerformanceTestCase, observation: Observation) -> tuple[list[ percentile=rank, observed=observe(metric, rank, observation, case.background_cap_exempt_tables), limit=limit.max if limit else None, + floor=limit.min if limit else None, detail=_detail(metric, observation, case.background_cap_exempt_tables), breakdown=breakdown(metric, rank, observation, case.background_cap_exempt_tables), ) ) - exceeded: Final = [ - f"{row.label}: observed {row.observed:.4f} > limit {row.limit:.4f}{f' ({row.detail})' if row.detail else ''}" - if row.over and row.observed is not None and row.limit is not None - else f"{row.label}: limit set but nothing observed" - for row in rows - if row.over or row.missing - ] + exceeded: Final = [_breach(row) for row in rows if row.over or row.missing] return rows, exceeded +def _breach(row: Row) -> str: + if row.observed is None: + return f"{row.label}: limit set but nothing observed" + if row.floor is not None: + return f"{row.label}: observed {row.observed:.4f} < floor {row.floor:.4f}" + return f"{row.label}: observed {row.observed:.4f} > limit {row.bound}{f' ({row.detail})' if row.detail else ''}" + + def _detail(metric: Metric, observation: Observation, exempt_tables: frozenset[str]) -> str: if metric == "postgres_max_statement_per_request": busiest: Final = observation.busiest_awaited_statement @@ -516,12 +604,13 @@ def _detail(metric: Metric, observation: Observation, exempt_tables: frozenset[s def _row_properties(row: Row) -> tuple[tuple[str, float | str], ...]: observed: Final = ((row.property_name, row.observed),) if row.observed is not None else () limit: Final = ((f"{row.property_name}_limit", row.limit),) if row.limit is not None else () + floor: Final = ((f"{row.property_name}_floor", row.floor),) if row.floor is not None else () contributors: Final = ( ((f"{row.property_name}_breakdown", json.dumps([asdict(item) for item in row.breakdown])),) if row.breakdown else () ) - return observed + limit + contributors + return observed + limit + floor + contributors def junit_properties(rows: Iterable[Row]) -> tuple[tuple[str, float | str], ...]: @@ -546,9 +635,17 @@ def render(case: PerformanceTestCase, observation: Observation, rows: list[Row]) ] for row in rows: observed = "n/a" if row.observed is None else f"{row.observed:.4f}" - limit = "" if row.limit is None else f"{row.limit:.4f}" - flag = " OVER" if row.over else " MISSING" if row.missing else "" - lines.append(f" {row.label:32} {observed:>12} {limit:>12}{flag}") + flag = ( + " UNDER" + if row.over and row.floor is not None + else " OVER" + if row.over + else " MISSING" + if row.missing + else "" + ) + lines.append(f" {row.label:32} {observed:>12} {row.bound:>12}{flag}") + lines.extend(render_steps(observation.steps())) lines.extend(_diagnostics(observation)) lines.extend(_datastore_lines(observation)) for record in failures: diff --git a/tests/e2e/perf/setup/performance_test_case.py b/tests/e2e/perf/setup/performance_test_case.py index 3f39b4c38c8..512ef8b911f 100644 --- a/tests/e2e/perf/setup/performance_test_case.py +++ b/tests/e2e/perf/setup/performance_test_case.py @@ -9,8 +9,10 @@ so a case is a literal a reviewer can read top to bottom. from __future__ import annotations +import bisect from collections.abc import Mapping -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace +from itertools import accumulate from typing import Final, Literal from setup.event_loop import EventLoopStalls @@ -36,6 +38,7 @@ Metric = Literal[ "postgres_active_sessions", "postgres_slowest_statement_seconds", "postgres_peak_connections", + "sustained_rps", ] @@ -97,12 +100,50 @@ SHORT: Final = RequestShape("short", input_tokens=(5_000, 100_000), tools=True) LONG: Final = RequestShape("long", input_tokens=(300_000, 1_000_000), tools=True) +@dataclass(frozen=True, slots=True) +class Ramp: + """Rate stepped up through the run: `first_rps`, then `step_rps` more every `step_seconds`, up to `last_rps`. + Each step is scored on its own and the run reports `sustained_rps`, the last step the pod held before the + first one it did not: a step holds when its `gateway_added_seconds` p99 stayed under `held_p99_seconds`, it + completed at least `HELD_SHARE` of its rate and no more than `HELD_UNEXPECTED_SHARE` of its requests came back + with an unexpected status (`setup/ramp.py`). Requests are sent on a schedule drawn from the plan's seed, not + from per-user intervals, so the rate changes on the step boundary.""" + + first_rps: float + last_rps: float + step_rps: float + step_seconds: float + held_p99_seconds: float + + @property + def rates(self) -> tuple[float, ...]: + count: Final = round((self.last_rps - self.first_rps) / self.step_rps) + 1 + return tuple(self.first_rps + index * self.step_rps for index in range(count)) + + @property + def counts(self) -> tuple[int, ...]: + return tuple(round(rate * self.step_seconds) for rate in self.rates) + + @property + def total_requests(self) -> int: + return sum(self.counts) + + @property + def seconds(self) -> float: + return len(self.rates) * self.step_seconds + + def step_of(self, seq: int) -> int: + """Zero-based step of the request with 1-based plan number `seq`.""" + return bisect.bisect_left(tuple(accumulate(self.counts)), seq) + + @dataclass(frozen=True, slots=True) class Traffic: """The plan: `total_requests` drawn from `seed`, sent at `rps` by `concurrency` simulated - users. Shares are fractions of the requests: `endpoints` and `shapes` must each sum to 1, - `intervene_share` picks non-streaming requests the guardrail rewrites, `block_share` picks - requests it blocks.""" + users, or stepped up through a `ramp` (then `rps` is its first step and `total_requests` its + total, see `ramped`). Shares are fractions of the requests: `endpoints` and `shapes` must each + sum to 1, `intervene_share` picks non-streaming requests the guardrail rewrites, `block_share` + picks requests it blocks.""" seed: int rps: float @@ -113,20 +154,33 @@ class Traffic: intervene_share: float = 0.0 block_share: float = 0.0 concurrency: int = 20 + ramp: Ramp | None = None @property def expected_seconds(self) -> float: - return self.total_requests / self.rps + return self.total_requests / self.rps if self.ramp is None else self.ramp.seconds + + +def ramped(base: Traffic, ramp: Ramp, *, seed: int, concurrency: int) -> Traffic: + """`base`'s request mix sent along `ramp` instead of at one rate. `concurrency` caps the requests in + flight, so size it past the deepest step the ramp is expected to reach.""" + return replace( + base, seed=seed, rps=ramp.first_rps, total_requests=ramp.total_requests, concurrency=concurrency, ramp=ramp + ) @dataclass(frozen=True, slots=True) class Limit: - """One pass/fail rule: `metric` (at `percentile`, or the whole-run value) must not exceed `max`. - Metrics without a Limit are still reported.""" + """One pass/fail rule: `metric` (at `percentile`, or the whole-run value) must not exceed `max`, or must + reach `min` for a floor such as `sustained_rps`. Metrics without a Limit are still reported.""" metric: Metric - max: float + max: float | None = None percentile: int | None = None + min: float | None = None + + def __post_init__(self) -> None: + assert (self.max is None) != (self.min is None), f"{self.metric}: a Limit is either a max or a min" @property def label(self) -> str: @@ -201,6 +255,14 @@ class PerformanceTestCase: assert 0 <= self.traffic.stream_share <= 1, f"{self.name}: stream_share out of range" assert self.traffic.intervene_share + self.traffic.block_share <= 1, f"{self.name}: guardrail shares exceed 1" assert self.traffic.total_requests > 0 and self.traffic.rps > 0, f"{self.name}: empty traffic" + ramp: Final = self.traffic.ramp + assert ramp is None or (ramp.first_rps, ramp.total_requests) == ( + self.traffic.rps, + self.traffic.total_requests, + ), f"{self.name}: traffic rps and total_requests must be the ramp's, build it with ramped()" + assert ramp is None or (0 < ramp.first_rps <= ramp.last_rps and ramp.step_rps > 0 and ramp.step_seconds > 0), ( + f"{self.name}: ramp {ramp} does not step up" + ) assert self.model_groups, f"{self.name}: no model groups" assert 0 <= self.mock_latency.jitter_seconds <= self.mock_latency.seconds or self.mock_latency.seconds == 0, ( f"{self.name}: mock latency jitter {self.mock_latency.jitter_seconds} s exceeds {self.mock_latency.seconds} s" diff --git a/tests/e2e/perf/setup/plan.py b/tests/e2e/perf/setup/plan.py index 6b9a5a19662..4f28736a8e9 100644 --- a/tests/e2e/perf/setup/plan.py +++ b/tests/e2e/perf/setup/plan.py @@ -12,6 +12,7 @@ from dataclasses import dataclass from typing import Final, Literal, TypeVar from setup.performance_test_case import Endpoint, PerformanceTestCase, RequestShape, Team +from setup.ramp import send_offsets Flag = Literal["n", "i", "b"] CallerKind = Literal["jwt", "key"] @@ -39,6 +40,7 @@ class PlannedRequest: flag: Flag shape: RequestShape input_tokens: int + send_at_s: float | None = None @property def blocked(self) -> bool: @@ -98,6 +100,7 @@ def build_plan(case: PerformanceTestCase) -> tuple[PlannedRequest, ...]: case.validate() rng: Final = random.Random(case.traffic.seed) callers: Final = _callers_by_group(case) + offsets: Final = None if case.traffic.ramp is None else send_offsets(case.traffic.ramp, case.traffic.seed) planned: list[PlannedRequest] = [] for seq in range(1, case.traffic.total_requests + 1): endpoint = _pick(rng, case.traffic.endpoints) @@ -117,6 +120,7 @@ def build_plan(case: PerformanceTestCase) -> tuple[PlannedRequest, ...]: flag=flag, shape=shape, input_tokens=rng.randint(low, high), + send_at_s=None if offsets is None else offsets[seq - 1], ) ) return tuple(planned) diff --git a/tests/e2e/perf/setup/podcpu.py b/tests/e2e/perf/setup/podcpu.py index 400e5b7d38b..865fa14efe0 100644 --- a/tests/e2e/perf/setup/podcpu.py +++ b/tests/e2e/perf/setup/podcpu.py @@ -19,6 +19,8 @@ from urllib.parse import quote from e2e_http import AuthHeaders, Success, get_external from pydantic import BaseModel +from setup.series import Recorder, Series + SELECTOR_ENV: Final = "E2E_PERF_GATEWAY_POD_SELECTOR" _SERVICE_ACCOUNT: Final = Path("/var/run/secrets/kubernetes.io/serviceaccount") _INTERVAL_S: Final = 5.0 @@ -49,7 +51,8 @@ def cores(quantity: str) -> float: class GatewayCpuSampler: - """Integrates the matching pods' core use over the time it is running; `seconds` is CPU seconds.""" + """Integrates the matching pods' core use over the time it is running; `seconds` is CPU seconds and + `series` the CPU seconds of each sampling interval, stamped when the interval ended.""" def __init__(self, selector: str, namespace: str, token: str, ca_bundle: str) -> None: self._url: Final = ( @@ -60,12 +63,15 @@ class GatewayCpuSampler: self._ca_bundle: Final = ca_bundle self._stop: Final = threading.Event() self._thread: Final = threading.Thread(target=self._run, name="gateway-cpu-sampler", daemon=True) - self._cpu_seconds = 0.0 - self._samples = 0 + self._intervals: Final[Recorder[float]] = Recorder() @property def seconds(self) -> float | None: - return self._cpu_seconds if self._samples > 0 else None + return sum(sample.value for sample in self._intervals.series.samples) if len(self._intervals) else None + + @property + def series(self) -> Series[float]: + return self._intervals.series def _read_cores(self) -> float | None: result: Final = get_external( @@ -81,8 +87,7 @@ class GatewayCpuSampler: now: float = time.monotonic() current: float | None = self._read_cores() if current is not None: - self._cpu_seconds += current * (now - last) - self._samples += 1 + self._intervals.record(current * (now - last)) last = now # rebind-ok: the sampling clock advances every tick def start(self) -> None: @@ -95,6 +100,7 @@ class GatewayCpuSampler: class NoSampler: seconds: Final[None] = None + series: Final[None] = None def start(self) -> None: return None diff --git a/tests/e2e/perf/setup/ramp.py b/tests/e2e/perf/setup/ramp.py new file mode 100644 index 00000000000..217e140d470 --- /dev/null +++ b/tests/e2e/perf/setup/ramp.py @@ -0,0 +1,116 @@ +"""A ramped run step by step: the send schedule its plan carries (`send_offsets`), one `StepReading` per step +and `sustained_rps`, the last step the pod held before the first it did not. `Ramp` itself sits in +`performance_test_case.py` with the rest of the case schema.""" + +from __future__ import annotations + +import random +from collections.abc import Iterable, Iterator +from dataclasses import dataclass +from itertools import chain, takewhile +from typing import Final + +from setup.performance_test_case import Ramp + +HELD_SHARE: Final = 0.9 +"""A step that completed fewer than this share of its rate is a queue that stopped draining.""" +HELD_UNEXPECTED_SHARE: Final = 0.01 +"""A step with more than this share of unexpected statuses broke, whatever its latency.""" +_JITTER: Final = 0.25 + + +def _step_offsets(ramp: Ramp, index: int, count: int, rng: random.Random) -> Iterator[float]: + for slot in range(count): + yield (index + (slot + 0.5 + rng.uniform(-_JITTER, _JITTER)) / count) * ramp.step_seconds + + +def send_offsets(ramp: Ramp, seed: int) -> tuple[float, ...]: + """Seconds after the run starts to send each request of the plan, in plan order: a step's requests spread + evenly over its window, each moved by up to a quarter of a slot either way, drawn from `seed` so a rerun + sends at the same seconds.""" + rng: Final = random.Random(f"ramp-{seed}") + return tuple(chain.from_iterable(_step_offsets(ramp, index, count, rng) for index, count in enumerate(ramp.counts))) + + +@dataclass(frozen=True, slots=True) +class StepReading: + rps: float + planned: int + completed_rps: float + gateway_added_p99: float | None + unexpected: int + held_p99_seconds: float + gateway_added_p50: float | None = None + time_to_first_token_p99: float | None = None + cpu_seconds_per_request: float | None = None + redis_latency_mean_seconds: float | None = None + postgres_peak_connections: int | None = None + """The hold verdict reads the first six. The rest say what the pod was doing on that step: CPU per request + from the pod sampler, Redis latency from the gateway's `/metrics`, connections from `pg_stat_activity`, + all cut to the step's window, so a latency jump can be told apart as CPU, waiting or neither.""" + + @property + def held(self) -> bool: + return ( + self.gateway_added_p99 is not None + and self.gateway_added_p99 <= self.held_p99_seconds + and self.completed_rps >= HELD_SHARE * self.rps + and self.unexpected <= HELD_UNEXPECTED_SHARE * self.planned + ) + + +def sustained_rps(steps: Iterable[StepReading]) -> float: + """The rate of the last step that held before the first that did not, 0 when the first step did not hold.""" + held: Final = tuple(takewhile(lambda step: step.held, steps)) + return held[-1].rps if held else 0.0 + + +def step_properties(steps: Iterable[StepReading]) -> tuple[tuple[str, float], ...]: + """The per-step readings as JUnit properties (`perf_ramp_step__`, every reading the step has) + plus the hold bound, so the dashboard can draw the run's curves next to the sustained rate.""" + rows: Final = tuple(steps) + hold: Final = (("perf_ramp_hold_p99_seconds", rows[0].held_p99_seconds),) if rows else () + return (*hold, *chain.from_iterable(_step_properties(step) for step in rows)) + + +def _step_properties(step: StepReading) -> Iterator[tuple[str, float]]: + prefix: Final = f"perf_ramp_step_{step.rps:g}" + readings: Final[tuple[tuple[str, float | None], ...]] = ( + ("gateway_added_p99", step.gateway_added_p99), + ("completed_rps", step.completed_rps), + ("unexpected", float(step.unexpected)), + ("gateway_added_p50", step.gateway_added_p50), + ("time_to_first_token_p99", step.time_to_first_token_p99), + ("cpu_seconds_per_request", step.cpu_seconds_per_request), + ("redis_latency_mean_seconds", step.redis_latency_mean_seconds), + ( + "postgres_peak_connections", + None if step.postgres_peak_connections is None else float(step.postgres_peak_connections), + ), + ) + return ((f"{prefix}_{name}", value) for name, value in readings if value is not None) + + +def render_steps(steps: Iterable[StepReading]) -> tuple[str, ...]: + rows: Final = tuple(steps) + if not rows: + return () + header: Final = ( + f" steps: {'rps':>6} {'planned':>8} {'completed rps':>14} {'gateway added p50':>18} {'p99':>8}" + f" {'ttft p99':>9} {'unexpected':>11} {'cpu s/req':>10} {'redis mean':>11} {'pg conns':>9}" + ) + return (header, *(_step_line(step) for step in rows)) + + +def _seconds(value: float | None) -> str: + return "n/a" if value is None else f"{value:.4f}" + + +def _step_line(step: StepReading) -> str: + conns: Final = "n/a" if step.postgres_peak_connections is None else str(step.postgres_peak_connections) + return ( + f" {step.rps:6.1f} {step.planned:8d} {step.completed_rps:14.2f} {_seconds(step.gateway_added_p50):>18}" + f" {_seconds(step.gateway_added_p99):>8} {_seconds(step.time_to_first_token_p99):>9} {step.unexpected:11d}" + f" {_seconds(step.cpu_seconds_per_request):>10} {_seconds(step.redis_latency_mean_seconds):>11} {conns:>9}" + f" {'held' if step.held else 'broke'}" + ) diff --git a/tests/e2e/perf/setup/run.py b/tests/e2e/perf/setup/run.py index 272c64754c4..ec2927e90af 100644 --- a/tests/e2e/perf/setup/run.py +++ b/tests/e2e/perf/setup/run.py @@ -63,6 +63,7 @@ def wire_request(item: PlannedRequest, setup: CaseSetup) -> WireRequest: token_source=setup.token_source(item.caller), stream=item.stream, blocked=item.blocked, + send_at_s=item.send_at_s, ) diff --git a/tests/e2e/perf/setup/series.py b/tests/e2e/perf/setup/series.py new file mode 100644 index 00000000000..78e5cf272e1 --- /dev/null +++ b/tests/e2e/perf/setup/series.py @@ -0,0 +1,48 @@ +"""Readings a background sampler takes while the traffic runs, each stamped with the wall clock +(`time.time()`, the clock the Locust records carry in `started_at`), so a ramped run can cut them into its +steps' windows after the fact.""" + +from __future__ import annotations + +import time +from dataclasses import dataclass +from typing import Final, Generic, TypeVar + +T = TypeVar("T") + + +@dataclass(frozen=True, slots=True) +class Sample(Generic[T]): + at: float + value: T + + +@dataclass(frozen=True, slots=True) +class Series(Generic[T]): + samples: tuple[Sample[T], ...] + + def between(self, start: float, end: float) -> tuple[T, ...]: + """The values stamped inside `[start, end)`, in time order.""" + return tuple(sample.value for sample in self.samples if start <= sample.at < end) + + def last_before(self, at: float) -> T | None: + """The newest value stamped before `at`, None when the series starts later.""" + earlier: Final = tuple(sample.value for sample in self.samples if sample.at < at) + return earlier[-1] if earlier else None + + +class Recorder(Generic[T]): + """What a sampler thread appends to; `series` is a frozen copy for the observation.""" + + def __init__(self) -> None: + self._samples: list[Sample[T]] = [] + + def record(self, value: T) -> None: + self._samples.append(Sample(at=time.time(), value=value)) + + @property + def series(self) -> Series[T]: + return Series(samples=tuple(self._samples)) + + def __len__(self) -> int: + return len(self._samples) diff --git a/tests/e2e/perf/setup/telemetry.py b/tests/e2e/perf/setup/telemetry.py index e52bcabafda..185cc788601 100644 --- a/tests/e2e/perf/setup/telemetry.py +++ b/tests/e2e/perf/setup/telemetry.py @@ -1,5 +1,5 @@ -"""The gateway's own `/metrics`, read before and after a run for the numbers nothing outside the gateway -can see. Today that is `litellm_redis_latency`, the histogram customers' Grafana panels show when Redis +"""The gateway's own `/metrics`, read before and after a run (and every few seconds during a ramped run, see +`telemetry_sampler`) for the numbers nothing outside the gateway can see. Today that is `litellm_redis_latency`, the histogram customers' Grafana panels show when Redis slows the gateway down. The gateway's latency and request counts are read per request from its response headers instead (`setup/wire.py`), which gives percentiles where a histogram delta only gives a mean. @@ -10,7 +10,9 @@ on psutil (`proxy_usage.py`).""" from __future__ import annotations import math -from collections.abc import Iterable +import threading +from collections.abc import Generator, Iterable +from contextlib import contextmanager from dataclasses import dataclass from itertools import chain from typing import Final @@ -20,6 +22,10 @@ from prometheus_client.parser import text_string_to_metric_families from prometheus_client.samples import Sample from proxy_client import ProxyClient +from setup.series import Recorder, Series + +_SAMPLE_SECONDS: Final = 10.0 + @dataclass(frozen=True, slots=True) class HistogramTotals: @@ -63,3 +69,42 @@ def read_gateway_telemetry(proxy: ProxyClient) -> GatewayTelemetry | None: case every limit on a gateway-reported metric fails as missing rather than reading a silent zero.""" scrape: Final = proxy.probe("/metrics", params=NoBody()) return parse_gateway_telemetry(scrape.body) if scrape.status_code == 200 else None + + +class TelemetrySampler: + """`/metrics` every `_SAMPLE_SECONDS` on a background thread while traffic runs, so a ramped run can read + the Redis latency of each step's window as the difference between the scrapes around it.""" + + def __init__(self, proxy: ProxyClient) -> None: + self._proxy: Final = proxy + self._stop: Final = threading.Event() + self._thread: Final = threading.Thread(target=self._run, name="gateway-telemetry", daemon=True) + self._scrapes: Final[Recorder[GatewayTelemetry]] = Recorder() + + @property + def series(self) -> Series[GatewayTelemetry]: + return self._scrapes.series + + def _run(self) -> None: + while not self._stop.is_set(): + scrape: GatewayTelemetry | None = read_gateway_telemetry(self._proxy) + if scrape is not None: + self._scrapes.record(scrape) + _ = self._stop.wait(_SAMPLE_SECONDS) + + def start(self) -> None: + self._thread.start() + + def stop(self) -> None: + self._stop.set() + self._thread.join(timeout=30) + + +@contextmanager +def telemetry_sampler(proxy: ProxyClient) -> Generator[TelemetrySampler]: + sampler: Final = TelemetrySampler(proxy) + sampler.start() + try: + yield sampler + finally: + sampler.stop() diff --git a/tests/e2e/perf/setup/upgrade.py b/tests/e2e/perf/setup/upgrade.py index 36efc16e1fe..c9114ccbff2 100644 --- a/tests/e2e/perf/setup/upgrade.py +++ b/tests/e2e/perf/setup/upgrade.py @@ -145,7 +145,9 @@ class UpgradeMonitor: self.clock: Final = clock self.wall_clock: Final = wall_clock self.started: Final = clock() - self.status_limit: Final = next(limit.max for limit in CASE.limits if limit.metric == "unexpected_status_count") + self.status_limit: Final = next( + limit.max for limit in CASE.limits if limit.metric == "unexpected_status_count" and limit.max is not None + ) self.pending: tuple[WireRecord, ...] = () self.processed = 0 self.successes = 0 diff --git a/tests/e2e/perf/setup/wire.py b/tests/e2e/perf/setup/wire.py index 18351c18c6b..a1833fbd44b 100644 --- a/tests/e2e/perf/setup/wire.py +++ b/tests/e2e/perf/setup/wire.py @@ -51,6 +51,8 @@ class WireRequest(BaseModel): token_source: TokenSource | None = None stream: bool blocked: bool + send_at_s: float | None = None + """Seconds after the run starts to send this request, set when the case ramps; otherwise the users pace.""" class WireRecord(BaseModel): diff --git a/tests/e2e/perf/test_perf_cases.py b/tests/e2e/perf/test_perf_cases.py index 3e5c3c522c8..db657a04b00 100644 --- a/tests/e2e/perf/test_perf_cases.py +++ b/tests/e2e/perf/test_perf_cases.py @@ -44,8 +44,9 @@ from setup.performance_test_case import MockLatency, PerformanceTestCase from setup.plan import PlannedRequest, build_plan from setup.podcpu import gateway_cpu_sampler from setup.provision import MOCK_PROVIDERS, build_setup, guardrail_url, mock_url +from setup.ramp import step_properties from setup.run import run_plan -from setup.telemetry import read_gateway_telemetry +from setup.telemetry import read_gateway_telemetry, telemetry_sampler pytestmark = [pytest.mark.perf, pytest.mark.quiet_stack] _SPEND_SAMPLE: Final = 40 @@ -115,7 +116,12 @@ def test_perf_case( redis_before: Final = read_redis_counters() run_started: Final = time.monotonic() traffic_started_at: Final = datetime.now(tz=UTC) - with gateway_cpu_sampler() as cpu, redis_monitor() as monitor, connections_sampler() as connections: + with ( + gateway_cpu_sampler() as cpu, + redis_monitor() as monitor, + connections_sampler() as connections, + telemetry_sampler(proxy) as telemetry, + ): outcome: Final = run_plan(case, plan, setup) run_finished: Final = time.monotonic() db_after: Final = read_db_counters() @@ -151,6 +157,10 @@ def test_perf_case( telemetry_before=telemetry_before, telemetry_after=telemetry_after, peak_connections=connections.peak, + ramp=case.traffic.ramp, + cpu_series=cpu.series, + connection_series=connections.series, + telemetry_series=telemetry.series, ) rows, exceeded_limits = evaluate(case, observation) stall_rows: Final = _event_loop_rows(case, traffic_started_at) if case.event_loop is not None else () @@ -162,6 +172,7 @@ def test_perf_case( item.user_properties.extend( stall_properties(case.event_loop, stall_rows) if case.event_loop is not None else junit_properties(rows) ) + item.user_properties.extend(step_properties(observation.steps())) print(render(case, observation, rows)) if case.event_loop is not None: print(render_stalls(case.event_loop, stall_rows)) diff --git a/tests/e2e_harness/perf/setup/test_ramp.py b/tests/e2e_harness/perf/setup/test_ramp.py new file mode 100644 index 00000000000..6d6c1640b59 --- /dev/null +++ b/tests/e2e_harness/perf/setup/test_ramp.py @@ -0,0 +1,168 @@ +from __future__ import annotations + +import math +from dataclasses import replace +from typing import Final + +from cases.small_bodies_5rps import CASE +from memory_readings import RssCapture +from setup.observe import Observation, evaluate, junit_properties +from setup.performance_test_case import Limit, Ramp, ramped +from setup.plan import build_plan +from setup.ramp import StepReading, send_offsets, step_properties, sustained_rps +from setup.series import Sample, Series +from setup.telemetry import GatewayTelemetry, HistogramTotals +from setup.wire import WireRecord + +NO_RSS: Final = RssCapture(readings=(), failures=()) + + +def test_send_offsets_put_each_step_s_requests_inside_its_window_in_plan_order() -> None: + ramp: Final = Ramp(first_rps=10.0, last_rps=30.0, step_rps=10.0, step_seconds=10.0, held_p99_seconds=5.0) + offsets: Final = send_offsets(ramp, seed=13) + assert ramp.counts == (100, 200, 300) + assert len(offsets) == ramp.total_requests + assert offsets == tuple(sorted(offsets)) + for seq, offset in enumerate(offsets, start=1): + step = ramp.step_of(seq) + assert step * 10.0 <= offset < (step + 1) * 10.0, f"request {seq} at {offset:.2f}s is outside step {step}" + assert send_offsets(ramp, seed=13) == offsets + assert send_offsets(ramp, seed=14) != offsets + + +def test_build_plan_carries_the_schedule_only_for_a_ramped_case() -> None: + ramp: Final = Ramp(first_rps=2.0, last_rps=4.0, step_rps=2.0, step_seconds=5.0, held_p99_seconds=5.0) + ramped_case: Final = replace(CASE, traffic=ramped(CASE.traffic, ramp, seed=3, concurrency=20)) + assert ramped_case.traffic.expected_seconds == 10.0 + assert [item.send_at_s for item in build_plan(ramped_case)] == list(send_offsets(ramp, seed=3)) + assert all(item.send_at_s is None for item in build_plan(CASE)) + + +def _step(rps: float, p99: float, completed_rps: float | None = None, unexpected: int = 0) -> StepReading: + return StepReading( + rps=rps, + planned=int(rps * 90), + completed_rps=rps if completed_rps is None else completed_rps, + gateway_added_p99=p99, + unexpected=unexpected, + held_p99_seconds=5.0, + ) + + +def test_sustained_rps_is_the_last_step_held_before_the_first_that_broke() -> None: + assert sustained_rps((_step(10, 0.5), _step(20, 1.0), _step(30, 9.0), _step(40, 0.7))) == 20 + assert sustained_rps((_step(10, 0.5), _step(20, 1.0, completed_rps=17.0), _step(30, 1.0))) == 10 + assert sustained_rps((_step(10, 0.5), _step(20, 1.0, unexpected=19), _step(30, 1.0))) == 10 + assert sustained_rps((_step(10, 0.5), _step(20, 1.0, unexpected=18), _step(30, 1.0))) == 30 + assert sustained_rps((_step(10, 6.0), _step(20, 1.0))) == 0 + + +def _record(seq: int, started_at: float, total_s: float) -> WireRecord: + return WireRecord( + seq=seq, + marker=f"perfreq-{seq:06d}-n", + caller="key-a1", + endpoint="chat_completions", + stream=False, + blocked=False, + started_at=started_at, + status=200, + header_s=total_s, + first_byte_s=total_s, + first_token_s=None, + total_s=total_s, + text="", + ok=True, + mock_delay_s=1.0, + call_id=None, + error=None, + ) + + +def test_a_floor_fails_when_the_second_step_s_latency_breaks_it_and_the_properties_carry_the_floor() -> None: + ramp: Final = Ramp(first_rps=10.0, last_rps=20.0, step_rps=10.0, step_seconds=100.0, held_p99_seconds=2.0) + case: Final = replace( + CASE, + traffic=ramped(CASE.traffic, ramp, seed=1, concurrency=50), + limits=(Limit("sustained_rps", min=20),), + ) + offsets: Final = send_offsets(ramp, seed=1) + records: Final = tuple( + _record(seq, 1000.0 + offset, 1.5 if ramp.step_of(seq) == 0 else 4.0) + for seq, offset in enumerate(offsets, start=1) + ) + observation: Final = Observation( + records=records, + mock_requests=(), + guardrail_calls=(), + rss_before=NO_RSS, + rss_after=NO_RSS, + wall_seconds=200.0, + ramp=ramp, + ) + first, second = observation.steps() + assert first.held and first.gateway_added_p99 == 0.5 and first.completed_rps > 9.5 + assert not second.held and second.gateway_added_p99 == 3.0 + rows, exceeded = evaluate(case, observation) + assert exceeded == ["sustained_rps: observed 10.0000 < floor 20.0000"] + assert {("perf_sustained_rps", 10.0), ("perf_sustained_rps_floor", 20.0)} <= set(junit_properties(rows)) + assert dict(step_properties((first, second))) == { + "perf_ramp_hold_p99_seconds": 2.0, + "perf_ramp_step_10_gateway_added_p99": 0.5, + "perf_ramp_step_10_completed_rps": first.completed_rps, + "perf_ramp_step_10_unexpected": 0.0, + "perf_ramp_step_10_gateway_added_p50": 0.5, + "perf_ramp_step_20_gateway_added_p99": 3.0, + "perf_ramp_step_20_completed_rps": second.completed_rps, + "perf_ramp_step_20_unexpected": 0.0, + "perf_ramp_step_20_gateway_added_p50": 3.0, + } + assert not evaluate(replace(case, limits=(Limit("sustained_rps", min=10),)), observation)[1] + + +def test_each_step_reads_the_pod_s_cpu_redis_latency_and_connections_from_its_own_window() -> None: + ramp: Final = Ramp(first_rps=10.0, last_rps=20.0, step_rps=10.0, step_seconds=100.0, held_p99_seconds=2.0) + records: Final = tuple( + _record(seq, 1000.0 + offset, 1.5) for seq, offset in enumerate(send_offsets(ramp, seed=1), start=1) + ) + started: Final = min(record.started_at for record in records) + redis: Final = tuple( + Sample(at=started + at, value=GatewayTelemetry(redis_latency=HistogramTotals(total=total, count=count))) + for at, total, count in ((-1.0, 0.0, 0.0), (50.0, 1.0, 100.0), (99.0, 2.0, 200.0), (199.0, 42.0, 300.0)) + ) + observation: Final = Observation( + records=records, + mock_requests=(), + guardrail_calls=(), + rss_before=NO_RSS, + rss_after=NO_RSS, + wall_seconds=200.0, + ramp=ramp, + cpu_series=Series(samples=tuple(Sample(at=started + at, value=5.0) for at in (10.0, 60.0, 110.0, 160.0))), + connection_series=Series( + samples=tuple( + Sample(at=started + at, value=n) for at, n in ((1.0, 12), (50.0, 18), (150.0, 25), (199.0, 20)) + ) + ), + telemetry_series=Series(samples=redis), + ) + first, second = observation.steps() + first_completed: Final = round(first.completed_rps * ramp.step_seconds) + assert first.cpu_seconds_per_request is not None and math.isclose( + first.cpu_seconds_per_request, 10.0 / first_completed + ) + assert first.redis_latency_mean_seconds is not None and math.isclose(first.redis_latency_mean_seconds, 0.01) + assert first.postgres_peak_connections == 18 + assert second.redis_latency_mean_seconds is not None and math.isclose(second.redis_latency_mean_seconds, 0.4) + assert second.postgres_peak_connections == 25 + assert first.time_to_first_token_p99 is None + names: Final = {name for name, _ in step_properties((first, second)) if name.startswith("perf_ramp_step_10_")} + assert names == { + "perf_ramp_step_10_gateway_added_p99", + "perf_ramp_step_10_completed_rps", + "perf_ramp_step_10_unexpected", + "perf_ramp_step_10_gateway_added_p50", + "perf_ramp_step_10_cpu_seconds_per_request", + "perf_ramp_step_10_redis_latency_mean_seconds", + "perf_ramp_step_10_postgres_peak_connections", + } diff --git a/tests/e2e_harness/perf/test_upgrade.py b/tests/e2e_harness/perf/test_upgrade.py index 9a6c9d2af14..3414476ba4c 100644 --- a/tests/e2e_harness/perf/test_upgrade.py +++ b/tests/e2e_harness/perf/test_upgrade.py @@ -304,7 +304,9 @@ def test_invalid_200_is_not_an_allowed_non_200(failed: WireRecord) -> None: def _ceiling(metric: str, percentile: int | None = None) -> float: return next( - limit.max for limit in upgrade_case().limits if limit.metric == metric and limit.percentile == percentile + limit.max + for limit in upgrade_case().limits + if limit.metric == metric and limit.percentile == percentile and limit.max is not None ) From 7ae46f27687f8d927c4fc16d796af68eb8bda273 Mon Sep 17 00:00:00 2001 From: Kerry Lu Date: Tue, 20 Oct 2026 22:37:21 -0700 Subject: [PATCH 056/235] fix(timing): carry litellm_received_at across to a planted litellm_metadata bucket so the overhead header stays anchored at receive time (#47790) A guardrail hook plants a litellm_metadata key on a chat completions request after the proxy stamped litellm_received_at into metadata. get_litellm_metadata_from_kwargs prefers that key when both exist and only copied the user_api_key_* fields across, so the response timing headers lost their receive anchor and x-litellm-overhead-duration-ms reported 2 ms on /v1/chat/completions against 58 ms on /v1/responses for the same proxy work. The receive stamp now rides along with the user_api_key_* fields. Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/litellm_core_utils/core_helpers.py | 7 +++- .../litellm_core_utils/test_core_helpers.py | 38 ++++++++++++++++++- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index b7c6f805233..1d61ef8519a 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -329,11 +329,14 @@ def add_missing_spend_metadata_to_litellm_metadata(litellm_metadata: dict, metad Helper to get litellm metadata for spend tracking PATCH for issue where both `litellm_metadata` and `metadata` are present in the kwargs - and user_api_key values are in 'metadata'. + and user_api_key values are in 'metadata'. The proxy receive stamp rides along for the same + reason: a guardrail hook can plant `litellm_metadata` on a chat request after the proxy stamped + `metadata`, and the response timing headers anchor on that stamp. """ potential_spend_tracking_metadata_substring: Final = "user_api_key" for key, value in dict(metadata).items(): - if potential_spend_tracking_metadata_substring in key: + carry_receive_stamp = key == "litellm_received_at" and key not in litellm_metadata + if potential_spend_tracking_metadata_substring in key or carry_receive_stamp: litellm_metadata[key] = value return litellm_metadata diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 6459aa98e70..d66c37ba8c2 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -1,5 +1,6 @@ """Tests for litellm_core_utils.core_helpers module.""" +import datetime import json import logging import threading @@ -18,13 +19,13 @@ from litellm.litellm_core_utils.core_helpers import ( drop_params_env_flag, drop_params_flag, get_litellm_metadata_from_kwargs, - get_parent_otel_span_from_kwargs, get_or_create_metadata_bucket, + get_parent_otel_span_from_kwargs, get_provider_response_headers_from_hidden_params, map_finish_reason, normalize_drop_params, - process_response_headers, preserve_upstream_non_openai_attributes, + process_response_headers, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, @@ -685,3 +686,36 @@ def test_spend_metadata_merge_reads_request_metadata_that_is_a_read_only_mapping ) assert merged == {"user_api_key_hash": "hash", "model_group": "claude"} + + +class TestReceiveStampSurvivesASecondMetadataBucket: + """A guardrail hook can plant ``litellm_metadata`` on a chat request after the proxy stamped + ``litellm_received_at`` into ``metadata``. The resolved bucket must still carry the stamp, or + ``x-litellm-overhead-duration-ms`` starts at the router call and hides every pre-call cost.""" + + def test_receive_stamp_is_carried_into_litellm_metadata(self): + received_at = datetime.datetime(2026, 10, 21, 5, 0, tzinfo=datetime.timezone.utc) + kwargs = { + "litellm_params": { + "metadata": {"litellm_received_at": received_at, "user_api_key_user_id": "user-1"}, + "litellm_metadata": {"applied_guardrails": ["perf-pre"]}, + } + } + + resolved = get_litellm_metadata_from_kwargs(kwargs) + + assert resolved is kwargs["litellm_params"]["litellm_metadata"] + assert resolved["litellm_received_at"] is received_at + assert resolved["user_api_key_user_id"] == "user-1" + + def test_litellm_metadata_keeps_its_own_receive_stamp(self): + own = datetime.datetime(2026, 10, 21, 5, 0, tzinfo=datetime.timezone.utc) + other = datetime.datetime(2026, 10, 21, 5, 1, tzinfo=datetime.timezone.utc) + kwargs = { + "litellm_params": { + "metadata": {"litellm_received_at": other}, + "litellm_metadata": {"litellm_received_at": own}, + } + } + + assert get_litellm_metadata_from_kwargs(kwargs)["litellm_received_at"] is own From 230f653523fa55aa446e1a5b6469f0871cf7278d Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Tue, 20 Oct 2026 22:50:28 -0700 Subject: [PATCH 057/235] fix(router): normalize native Responses latency per output token (#47727) * fix(router): normalize native Responses latency per output token * fix(router): isolate normalized latency samples during upgrades --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- litellm/router_strategy/lowest_latency.py | 61 +++-- .../router_strategy/test_lowest_latency.py | 222 ++++++++++++++++-- .../test_router_routing_groups.py | 4 +- 3 files changed, 249 insertions(+), 38 deletions(-) diff --git a/litellm/router_strategy/lowest_latency.py b/litellm/router_strategy/lowest_latency.py index 83bb22435e6..28ed062f1ec 100644 --- a/litellm/router_strategy/lowest_latency.py +++ b/litellm/router_strategy/lowest_latency.py @@ -14,7 +14,9 @@ from litellm._internal_context import with_service_target from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, safe_divide_seconds +from litellm.responses.utils import ResponseAPILoggingUtils from litellm.router_utils.batch_utils import is_batch_retrieve_call_type +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import LiteLLMPydanticObjectBase if TYPE_CHECKING: @@ -86,7 +88,7 @@ class LowestLatencyLoggingHandler(CustomLogger): { {model_group}_map: { id: { - "latency": [..] + "latency_normalized": [..] f"{date:hour:minute}" : {"tpm": 34, "rpm": 3} } } @@ -116,11 +118,16 @@ class LowestLatencyLoggingHandler(CustomLogger): final_value: float = response_ms total_tokens = 0 - if isinstance(response_obj, ModelResponse): - _usage: Final = getattr(response_obj, "usage", None) + if isinstance(response_obj, (ModelResponse, ResponsesAPIResponse)): + _usage: Final = ( + ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_obj.usage) + if isinstance(response_obj, ResponsesAPIResponse) + else getattr(response_obj, "usage", None) + ) if _usage is not None: completion_tokens: Final = _usage.completion_tokens - total_tokens = _usage.total_tokens + if isinstance(response_obj, ModelResponse): + total_tokens = _usage.total_tokens # response_ms is already normalized to float seconds above response_seconds: Final = response_ms @@ -143,10 +150,12 @@ class LowestLatencyLoggingHandler(CustomLogger): request_count_dict[id] = {} ## Latency - if len(request_count_dict[id].get("latency", [])) < self.routing_args.max_latency_list_size: - request_count_dict[id].setdefault("latency", []).append(final_value) + if len(request_count_dict[id].get("latency_normalized", [])) < self.routing_args.max_latency_list_size: + request_count_dict[id].setdefault("latency_normalized", []).append(final_value) else: - request_count_dict[id]["latency"] = request_count_dict[id]["latency"][1:] + [final_value] + request_count_dict[id]["latency_normalized"] = request_count_dict[id]["latency_normalized"][1:] + [ + final_value + ] ## Time to first token if time_to_first_token is not None: @@ -212,7 +221,7 @@ class LowestLatencyLoggingHandler(CustomLogger): { {model_group}_map: { id: { - "latency": [..] + "latency_normalized": [..] f"{date:hour:minute}" : {"tpm": 34, "rpm": 3} } } @@ -225,10 +234,15 @@ class LowestLatencyLoggingHandler(CustomLogger): request_count_dict[id] = {} ## Latency - give 1000s penalty for failing - if len(request_count_dict[id].get("latency", [])) < self.routing_args.max_latency_list_size: - request_count_dict[id].setdefault("latency", []).append(1000.0) + if ( + len(request_count_dict[id].get("latency_normalized", [])) + < self.routing_args.max_latency_list_size + ): + request_count_dict[id].setdefault("latency_normalized", []).append(1000.0) else: - request_count_dict[id]["latency"] = request_count_dict[id]["latency"][1:] + [1000.0] + request_count_dict[id]["latency_normalized"] = request_count_dict[id]["latency_normalized"][ + 1: + ] + [1000.0] await self.router_cache.async_set_cache( key=latency_key, @@ -270,7 +284,7 @@ class LowestLatencyLoggingHandler(CustomLogger): { {model_group}_map: { id: { - "latency": [..] + "latency_normalized": [..] "time_to_first_token_seconds": [..] f"{date:hour:minute}" : {"tpm": 34, "rpm": 3} } @@ -300,11 +314,16 @@ class LowestLatencyLoggingHandler(CustomLogger): final_value: float = response_ms total_tokens = 0 - if isinstance(response_obj, ModelResponse): - _usage: Final = getattr(response_obj, "usage", None) + if isinstance(response_obj, (ModelResponse, ResponsesAPIResponse)): + _usage: Final = ( + ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(response_obj.usage) + if isinstance(response_obj, ResponsesAPIResponse) + else getattr(response_obj, "usage", None) + ) if _usage is not None: completion_tokens: Final = _usage.completion_tokens - total_tokens = _usage.total_tokens + if isinstance(response_obj, ModelResponse): + total_tokens = _usage.total_tokens # response_ms is already normalized to float seconds above response_seconds: Final = response_ms @@ -331,10 +350,12 @@ class LowestLatencyLoggingHandler(CustomLogger): request_count_dict[id] = {} ## Latency - if len(request_count_dict[id].get("latency", [])) < self.routing_args.max_latency_list_size: - request_count_dict[id].setdefault("latency", []).append(final_value) + if len(request_count_dict[id].get("latency_normalized", [])) < self.routing_args.max_latency_list_size: + request_count_dict[id].setdefault("latency_normalized", []).append(final_value) else: - request_count_dict[id]["latency"] = request_count_dict[id]["latency"][1:] + [final_value] + request_count_dict[id]["latency_normalized"] = request_count_dict[id]["latency_normalized"][1:] + [ + final_value + ] ## Time to first token if time_to_first_token is not None: @@ -403,7 +424,7 @@ class LowestLatencyLoggingHandler(CustomLogger): ## if healthy deployment not yet used if d["model_info"]["id"] not in all_deployments: all_deployments[d["model_info"]["id"]] = { - "latency": [0], + "latency_normalized": [0], precise_minute: {"tpm": 0, "rpm": 0}, } @@ -443,7 +464,7 @@ class LowestLatencyLoggingHandler(CustomLogger): or _deployment.get("model_info", {}).get("rpm", None) or float("inf") ) - item_latency = item_map.get("latency", []) + item_latency = item_map.get("latency_normalized", []) item_ttft_latency = item_map.get("time_to_first_token_seconds", []) item_rpm = item_map.get(precise_minute, {}).get("rpm", 0) item_tpm = item_map.get(precise_minute, {}).get("tpm", 0) diff --git a/tests/unit/router_strategy/test_lowest_latency.py b/tests/unit/router_strategy/test_lowest_latency.py index 0a3a92691ba..9af5def0d56 100644 --- a/tests/unit/router_strategy/test_lowest_latency.py +++ b/tests/unit/router_strategy/test_lowest_latency.py @@ -17,8 +17,10 @@ from pydantic import ValidationError import litellm from litellm.caching.caching import DualCache from litellm.caching.in_memory_cache import InMemoryCache +from litellm.responses.utils import ResponseAPILoggingUtils from litellm.router import Router from litellm.router_strategy.lowest_latency import LowestLatencyLoggingHandler, RoutingArgs +from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse DEPLOYMENT_ID = "9876" KWARGS = { @@ -43,7 +45,7 @@ def _embedding_response() -> litellm.EmbeddingResponse: def _recorded_latencies(cache: DualCache): cached = cache.get_cache(key="gemini-embedding-001_map") or {} - return cached.get(DEPLOYMENT_ID, {}).get("latency", []) + return cached.get(DEPLOYMENT_ID, {}).get("latency_normalized", []) def test_sync_embedding_latency_is_json_serializable(): @@ -69,7 +71,7 @@ def test_sync_embedding_latency_is_json_serializable(): ) assert latencies[-1] == pytest.approx(2.0) # the exact failure mode from production: redis cache sync json.dumps - json.dumps({"latency": latencies}) + json.dumps({"latency_normalized": latencies}) @pytest.mark.asyncio @@ -94,7 +96,7 @@ async def test_async_embedding_latency_is_json_serializable(): f"raw timedelta leaked into latency list: {latencies}" ) assert latencies[-1] == pytest.approx(3.0) - json.dumps({"latency": latencies}) + json.dumps({"latency_normalized": latencies}) def _chat_response(completion_tokens: int) -> litellm.ModelResponse: @@ -131,7 +133,7 @@ async def test_async_chat_latency_normalized_per_token(): latencies = _recorded_latencies(cache) assert latencies and latencies[-1] == pytest.approx(0.5) # 2s / 4 tokens - json.dumps({"latency": latencies}) + json.dumps({"latency_normalized": latencies}) @pytest.mark.asyncio @@ -151,7 +153,7 @@ async def test_async_chat_zero_completion_tokens_falls_back_to_seconds(): latencies = _recorded_latencies(cache) assert latencies and latencies[-1] == pytest.approx(3.0) assert not isinstance(latencies[-1], timedelta) - json.dumps({"latency": latencies}) + json.dumps({"latency_normalized": latencies}) def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): @@ -168,7 +170,7 @@ def test_sync_chat_zero_completion_tokens_falls_back_to_seconds(): latencies = _recorded_latencies(cache) assert latencies and latencies[-1] == pytest.approx(2.0) assert not isinstance(latencies[-1], timedelta) - json.dumps({"latency": latencies}) + json.dumps({"latency_normalized": latencies}) MODEL_GROUP = "gpt-4o-mini" @@ -362,8 +364,8 @@ async def test_ttft_percentile_does_not_change_non_streaming_routing(): cache.set_cache( key=f"{MODEL_GROUP}_map", value={ - FAST_TTFT_ID: {"latency": [1.0], "time_to_first_token_seconds": [0.1]}, - SLOW_TTFT_ID: {"latency": [0.2], "time_to_first_token_seconds": [1.5]}, + FAST_TTFT_ID: {"latency_normalized": [1.0], "time_to_first_token_seconds": [0.1]}, + SLOW_TTFT_ID: {"latency_normalized": [0.2], "time_to_first_token_seconds": [1.5]}, }, ) @@ -380,7 +382,7 @@ async def test_ttft_percentile_does_not_change_non_streaming_routing(): @pytest.mark.asyncio @pytest.mark.parametrize( "cached_entry", - [{"latency": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}], + [{"latency_normalized": []}, {"2026-09-05-15-39": {"tpm": 28, "rpm": 1}}], ids=["empty_latency_list", "minute_bucket_only_as_cost_based_routing_writes"], ) async def test_async_get_available_deployments_treats_missing_samples_as_zero_latency(cached_entry): @@ -388,7 +390,7 @@ async def test_async_get_available_deployments_treats_missing_samples_as_zero_la handler = LowestLatencyLoggingHandler(router_cache=cache) cache.set_cache( key="gemini-embedding-001_map", - value={DEPLOYMENT_ID: cached_entry, "slower": {"latency": [0.5]}}, + value={DEPLOYMENT_ID: cached_entry, "slower": {"latency_normalized": [0.5]}}, ) healthy_deployments = [ {"model_info": {"id": DEPLOYMENT_ID}, "litellm_params": {}}, @@ -605,7 +607,7 @@ def test_latency_updated(): end_time=end_time, ) latency_key = f"{model_group}_map" - assert end_time - start_time == test_cache.get_cache(key=latency_key)[deployment_id]["latency"][0] + assert end_time - start_time == test_cache.get_cache(key=latency_key)[deployment_id]["latency_normalized"][0] @pytest.mark.usefixtures("frozen_latency_clock") @@ -1002,7 +1004,7 @@ def test_latency_list_trimming_discards_oldest_entry(): latency_key = f"{model_group}_map" cached_data = test_cache.get_cache(key=latency_key) - latency_list = cached_data[deployment_id]["latency"] + latency_list = cached_data[deployment_id]["latency_normalized"] assert len(latency_list) == max_size, f"Expected {max_size} entries, got {len(latency_list)}" @@ -1062,7 +1064,7 @@ async def test_latency_list_trimming_discards_oldest_entry_async(): latency_key = f"{model_group}_map" cached_data = await test_cache.async_get_cache(key=latency_key) - latency_list = cached_data[deployment_id]["latency"] + latency_list = cached_data[deployment_id]["latency_normalized"] assert len(latency_list) == max_size @@ -1129,7 +1131,7 @@ async def test_timeout_penalty_discards_oldest_entry(): latency_key = f"{model_group}_map" cached_data = await test_cache.async_get_cache(key=latency_key) - latency_list = cached_data[deployment_id]["latency"] + latency_list = cached_data[deployment_id]["latency_normalized"] assert len(latency_list) == max_size @@ -1181,7 +1183,7 @@ def test_list_order_preserved_after_multiple_trims(): latency_key = f"{model_group}_map" cached_data = test_cache.get_cache(key=latency_key) - latency_list = cached_data[deployment_id]["latency"] + latency_list = cached_data[deployment_id]["latency_normalized"] assert len(latency_list) == max_size @@ -1423,7 +1425,7 @@ def test_latency_cache_honors_custom_ttl() -> None: latency_key: Final = "ttl-group_map" cached: Final = cache.get_cache(key=latency_key) assert isinstance(cached, dict) - assert cached["deployment"]["latency"] == [1.0] + assert cached["deployment"]["latency_normalized"] == [1.0] assert cache.in_memory_cache.ttl_dict[latency_key] == 103.0 clock.return_value = 104.0 @@ -1461,3 +1463,191 @@ async def test_router_model_group_usage_increases_by_logged_tokens() -> None: updated_usage: Final = await router.get_model_group_usage(model_group=model_group) assert updated_usage[0] == initial_usage[0] + 15 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_callback", [False, True]) +@pytest.mark.parametrize("normalized_usage", [False, True]) +@pytest.mark.parametrize("native_seconds", [2, 3]) +async def test_responses_and_chat_latency_use_comparable_token_units( + async_callback: bool, normalized_usage: bool, native_seconds: int +) -> None: + cache: Final = DualCache() + handler: Final = LowestLatencyLoggingHandler(router_cache=cache) + usage: Final = ResponseAPIUsage(input_tokens=10, output_tokens=100, total_tokens=110) + native: Final = ResponsesAPIResponse(id="resp-latency", created_at=0, output=[], usage=usage) + logged_native: Final = ( + native.model_copy(update={"usage": ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)}) + if normalized_usage + else native + ) + original_usage: Final = logged_native.usage + assert original_usage is not None + original_fields: Final = original_usage.model_dump() + start: Final = datetime(2026, 1, 1) + for deployment_id, response, seconds in ( + ("native", logged_native, native_seconds), + ("bridge", _chat_response(100), 3), + ): + kwargs: Final = { + "call_type": "aresponses", + "litellm_params": {"metadata": {"model_group": "mixed"}, "model_info": {"id": deployment_id}}, + } + if async_callback: + await handler.async_log_success_event(kwargs, response, start, start + timedelta(seconds=seconds)) + else: + handler.log_success_event(kwargs, response, start, start + timedelta(seconds=seconds)) + recorded: Final = cache.get_cache(key="mixed_map") + assert recorded["native"]["latency_normalized"] == pytest.approx([native_seconds / 100]) + assert recorded["bridge"]["latency_normalized"] == pytest.approx([3 / 100]) + assert logged_native.usage is original_usage + assert logged_native.usage.model_dump() == original_fields + assert native.usage is usage + deployments: Final = [ + {"model_name": "mixed", "model_info": {"id": name}, "litellm_params": {}} for name in ("native", "bridge") + ] + picked: Final = handler.get_available_deployments( + model_group="mixed", healthy_deployments=deployments, request_kwargs={"stream": False, "metadata": {}} + ) + assert picked["model_info"]["id"] in ({"native"} if native_seconds < 3 else {"native", "bridge"}) + native_counts: Final = tuple(value for key, value in recorded["native"].items() if key != "latency_normalized") + assert native_counts == ({"tpm": 0, "rpm": 1},) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_callback", [False, True]) +@pytest.mark.parametrize( + "response", + [ + litellm.ModelResponse(), + ResponsesAPIResponse(id="resp-missing", created_at=0, output=[], usage=None), + ResponsesAPIResponse( + id="resp-zero", created_at=0, output=[], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=0, total_tokens=10), + ), + ], +) +async def test_responses_without_output_tokens_keep_elapsed_seconds( + async_callback: bool, response: litellm.ModelResponse | ResponsesAPIResponse +) -> None: + cache: Final = DualCache() + handler: Final = LowestLatencyLoggingHandler(router_cache=cache) + original_usage: Final = getattr(response, "usage", None) + if async_callback: + await handler.async_log_success_event(KWARGS, response, 1.0, 3.0) + else: + handler.log_success_event(KWARGS, response, 1.0, 3.0) + assert _recorded_latencies(cache) == [2.0] + assert getattr(response, "usage", None) is original_usage + + +@pytest.mark.usefixtures("frozen_latency_clock") +@pytest.mark.asyncio +@pytest.mark.parametrize("async_mode", [False, True]) +@pytest.mark.parametrize("preserve_new_samples", [False, True]) +async def test_legacy_latency_does_not_starve_deployments_after_upgrade( + async_mode: bool, preserve_new_samples: bool +) -> None: + cache: Final = DualCache() + handler: Final = LowestLatencyLoggingHandler(router_cache=cache) + minute: Final = FROZEN_NOW.strftime("%Y-%m-%d-%H-%M") + legacy: Final = { + "native": {"latency": [2.0], "time_to_first_token_seconds": [0.1], minute: {"tpm": 0, "rpm": 1}}, + "bridge": {"latency": [0.03], "time_to_first_token_seconds": [0.2], minute: {"tpm": 110, "rpm": 1}}, + } + cache.set_cache(key="mixed_map", value=copy.deepcopy(legacy)) + deployments: Final = [ + {"model_name": "mixed", "model_info": {"id": name}, "litellm_params": {}} for name in ("native", "bridge") + ] + native: Final = ResponsesAPIResponse( + id="resp-upgrade", created_at=0, output=[], + usage=ResponseAPIUsage(input_tokens=10, output_tokens=100, total_tokens=110), + ) + for iteration in range(2): + bridge_kwargs: Final = { + "litellm_params": {"metadata": {"model_group": "mixed"}, "model_info": {"id": "bridge"}}, + } + if async_mode: + await handler.async_log_success_event(bridge_kwargs, _chat_response(100), 0.0, 3.0) + else: + handler.log_success_event(bridge_kwargs, _chat_response(100), 0.0, 3.0) + selected: Final = ( + await handler.async_get_available_deployments(model_group="mixed", healthy_deployments=deployments) + if async_mode + else handler.get_available_deployments(model_group="mixed", healthy_deployments=deployments) + ) + assert selected["model_info"]["id"] == "native" + native_kwargs: Final = { + "litellm_params": {"metadata": {"model_group": "mixed"}, "model_info": {"id": "native"}}, + } + if async_mode: + await handler.async_log_success_event(native_kwargs, native, 0.0, 2.0) + else: + handler.log_success_event(native_kwargs, native, 0.0, 2.0) + recorded: Final = copy.deepcopy(cache.get_cache(key="mixed_map")) + assert recorded["native"]["latency_normalized"] == pytest.approx( + [0.02] * (iteration + 1 if preserve_new_samples else 1) + ) + assert recorded["native"]["latency"] == [2.0] + assert recorded["native"]["time_to_first_token_seconds"] == [0.1] + assert recorded["bridge"]["time_to_first_token_seconds"] == [0.2] + assert recorded["native"][minute] == {"tpm": 0, "rpm": iteration + 2} + assert recorded["bridge"][minute] == {"tpm": 110 + (iteration + 1) * 110, "rpm": iteration + 2} + streaming: Final = handler.get_available_deployments( + model_group="mixed", healthy_deployments=deployments, request_kwargs={"stream": True} + ) + assert streaming["model_info"]["id"] == "native" + cache.set_cache( + key="mixed_map", + value={ + name: {**(recorded[name] if preserve_new_samples else legacy[name]), minute: recorded[name][minute]} + for name in ("native", "bridge") + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_mode", [False, True]) +async def test_normalized_latency_window_evicts_samples_without_changing_legacy_data(async_mode: bool) -> None: + cache: Final = DualCache() + cache.set_cache(key="gemini-embedding-001_map", value={DEPLOYMENT_ID: {"latency": [2.0]}}) + handler: Final = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"max_latency_list_size": 2}) + for elapsed, expected in ((2.0, [0.02]), (3.0, [0.02, 0.03]), (4.0, [0.03, 0.04])): + if async_mode: + await handler.async_log_success_event(KWARGS, _chat_response(100), 0.0, elapsed) + else: + handler.log_success_event(KWARGS, _chat_response(100), 0.0, elapsed) + recorded: Final = cache.get_cache(key="gemini-embedding-001_map")[DEPLOYMENT_ID] + assert recorded["latency_normalized"] == pytest.approx(expected) + assert recorded["latency"] == [2.0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("existing_samples", [(), (0.02, 0.03)]) +async def test_timeout_penalty_updates_normalized_latency_and_avoids_failed_deployment( + existing_samples: tuple[float, ...], +) -> None: + cache: Final = DualCache() + handler: Final = LowestLatencyLoggingHandler(router_cache=cache, routing_args={"max_latency_list_size": 2}) + cache.set_cache( + key="mixed_map", + value={ + "native": {"latency": [0.001], "latency_normalized": list(existing_samples)}, + "bridge": {"latency": [2.0], "latency_normalized": [0.04]}, + }, + ) + await handler.async_log_failure_event( + kwargs={ + "exception": litellm.Timeout(message="request timed out", model="test", llm_provider="openai"), + "litellm_params": {"metadata": {"model_group": "mixed"}, "model_info": {"id": "native"}}, + }, + response_obj=None, start_time=0.0, end_time=3.0, + ) + recorded: Final = cache.get_cache(key="mixed_map")["native"] + assert recorded["latency_normalized"] == [*existing_samples[-1:], 1000.0] + assert recorded["latency"] == [0.001] + selected: Final = await handler.async_get_available_deployments( + model_group="mixed", + healthy_deployments=[{"model_info": {"id": name}, "litellm_params": {}} for name in ("native", "bridge")], + ) + assert selected["model_info"]["id"] == "bridge" diff --git a/tests/unit/router_strategy/test_router_routing_groups.py b/tests/unit/router_strategy/test_router_routing_groups.py index 7e4b470f1c1..7235333f59f 100644 --- a/tests/unit/router_strategy/test_router_routing_groups.py +++ b/tests/unit/router_strategy/test_router_routing_groups.py @@ -1249,7 +1249,7 @@ async def test_latency_based_override_learns_from_the_overriding_requests(): ) def samples(deployment_id: str) -> list[float]: - recorded = (router.cache.get_cache("grp_map") or {}).get(deployment_id, {}).get("latency", []) + recorded = (router.cache.get_cache("grp_map") or {}).get(deployment_id, {}).get("latency_normalized", []) return [latency for latency in recorded if latency > 0] async def overriding_call() -> str: @@ -1316,7 +1316,7 @@ async def test_override_matching_a_routing_group_strategy_records_each_request_o deployment_id = response._hidden_params["model_id"] await _async_wait_until(lambda: (router.cache.get_cache("grp_map") or {}).get(deployment_id) is not None) - assert len(router.cache.get_cache("grp_map")[deployment_id]["latency"]) == 1 + assert len(router.cache.get_cache("grp_map")[deployment_id]["latency_normalized"]) == 1 assert request["litellm_logging_obj"].dynamic_success_callbacks is None From 66c8f5b1a9cefaa7c55b758f86f4ab552f2e859e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:59:32 +0000 Subject: [PATCH 058/235] fix(llms): join api_base and route with one shared helper (#47535) * fix(llms): sanitize trailing slashes in provider get_complete_url * test(llms): move trailing-slash url tests into each provider's mapped unit file * fix(llms): strip trailing slashes in Perplexity search and Ollama embedding URLs and run the Topaz tests in CI * fix(llms): strip trailing slashes in search, DashScope, VolcEngine, GigaChat and Gradient AI URL builders * fix(fastcrw): strip a trailing slash from api_base and expect one slash in the GigaChat tests * fix(gigachat): strip a trailing slash from the pass-through api_base * fix(llms): join api_base and route with one shared helper A deployment api_base that ends in / made several get_complete_url builders emit a double slash, which Envoy and other gateways answer with 404. Add join_api_base in url_utils and use it in those builders, with a regression test that drives every get_complete_url under litellm/llms with a trailing-slash api_base * refactor(llms): move per-provider api_base strips onto join_api_base Use the shared join_api_base helper in the search, chat, embedding, image variation, GigaChat and VolcEngine URL builders instead of a per-provider rstrip and suffix check, so one helper owns the single-slash join * ci: run the trailing-slash URL builder test in the llms shard * test(integration): cover trailing-slash api_base across provider URL builders Wire cells for chat, streaming, messages, responses, embeddings, search and pass-through with and without a trailing slash, an owned-proxy module for the flag-gated providers, env bases and a worker kill, two chaos bursts, and a Topaz SDK cell * fix(llms): send every provider route through join_api_base Every place under litellm/llms and litellm/main.py that joined a caller-settable api_base with a route by concatenation or f-string now goes through join_api_base, so an api_base with a trailing slash sends the same upstream path as one without it Routes made only of a caller-supplied id, model, or passthrough path stay on a plain join over an already normalized base, because the helper's suffix dedup would turn a lookup by id into a list or create call The sweep test now also reaches the factory-made JSON provider builders, and a new loopback test drives the public entrypoints with an api_base ending in /gw/ and checks that every request path keeps /gw/ with no double slash * fix(llms): keep the route skip only where a builder already had it join_api_base is a pure join again. The new ensure_api_route skips the route only when the URL path already ends in it, and it is used only where the merge base or the trailing-slash test PR head already skipped. Evals cancel, Predibase, and Azure AI threads append their route again, EdenAI model listing and Gemini RAG ingestion join through join_api_base, and Azure AI Anthropic checks the path so a host named anthropic-* builds /anthropic/v1/messages * fix(azure_ai): read /anthropic from the URL path when building the count_tokens URL * test(llms): run the Gemini RAG trailing-slash case through aingest The sync ingest wrapper needs a current event loop, which an earlier async test in the same worker clears * test(llms): cover the Azure realtime, Bedrock rerank, and oobabooga joins with a trailing-slash api_base * test(integration): cover api_base joins across providers, query strings and named hosts * feat(url_utils): add join_api_base and ensure_api_route helpers * fix(url_utils): keep the scheme slashes when a stripped suffix was the whole host join_api_base and ensure_api_route now keep the // of a base left with only a scheme, so http://v1 with /v1 stripped joins to http://v1/files as before. The Azure v1 client base_url goes through ensure_api_route so a base ending in /openai/v1 no longer doubles it * fix(azure_ai): build the count_tokens URL from the messages URL * fix(azure_ai): match anthropic as a whole path segment in the messages URL * fix(llms): keep main's Azure AI anthropic match and non-string Bedrock endpoints Azure AI chat and messages go back to cutting the base at the first /anthropic, as on main, and count_tokens keeps its path check, so a gateway path like /anthropic-proxy builds the same URLs as main. A base that already ends in /v1/messages still gets count_tokens next to it. A non-string aws_bedrock_runtime_endpoint (an httpx.URL VPC endpoint) is used as its text again in AgentCore, Bedrock responses and the RAGFlow base, instead of falling back to the default host. get_volcengine_base_url and GigaChat's get_api_base strip a trailing slash again for outside callers. The cache-hit wire test is skipped as a known proxy bug, since one cache hit writes two cache_hit spend rows. * fix(llms): drop a slash before the api_base query and tighten the route test checks * test(llms): annotate new test locals as Final and flatten the builder generator * fix(url_utils): keep a scheme-only api_base hostless so it fails before sending --------- Co-authored-by: Raushankumar0720 --- .github/workflows/test-unit.yml | 2 + litellm/litellm_core_utils/url_utils.py | 24 +- .../aiml/image_generation/transformation.py | 3 +- .../aiohttp_openai/chat/transformation.py | 8 +- .../llms/anthropic/batches/transformation.py | 12 +- litellm/llms/anthropic/common_utils.py | 13 +- .../anthropic/count_tokens/transformation.py | 3 +- litellm/llms/anthropic/files/handler.py | 6 +- .../llms/anthropic/files/transformation.py | 14 +- .../pass_through/messages/transformation.py | 6 +- .../llms/anthropic/skills/transformation.py | 10 +- litellm/llms/anthropic/wif.py | 3 +- .../llms/apiserpent/search/transformation.py | 7 +- .../audio_transcription/transformation.py | 4 +- litellm/llms/azure/common_utils.py | 3 +- litellm/llms/azure/realtime/handler.py | 4 +- .../azure/realtime/http_transformation.py | 5 +- litellm/llms/azure/search/transformation.py | 6 +- .../azure/text_to_speech/transformation.py | 11 +- litellm/llms/azure_ai/agents/handler.py | 17 +- .../anthropic/count_tokens/transformation.py | 18 +- .../anthropic/messages_transformation.py | 21 +- .../llms/azure_ai/anthropic/transformation.py | 11 + litellm/llms/azure_ai/embed/handler.py | 10 +- .../image_generation/flux_transformation.py | 4 +- .../image_generation/mai_transformation.py | 9 +- .../azure_ai/vector_stores/transformation.py | 4 +- .../llms/base_llm/decisions/transformation.py | 3 +- litellm/llms/base_llm/evals/transformation.py | 3 +- .../base_llm/realtime/http_transformation.py | 8 +- .../llms/base_llm/skills/transformation.py | 3 +- .../bedrock/chat/agentcore/transformation.py | 5 +- .../chat/chat_completions/transformation.py | 10 +- litellm/llms/bedrock/chat/converse_handler.py | 9 +- .../chat/invoke_agent/transformation.py | 9 +- .../amazon_openai_transformation.py | 10 +- .../base_invoke_transformation.py | 14 +- .../bedrock/claude_platform/common_utils.py | 5 +- litellm/llms/bedrock/common_utils.py | 6 +- .../bedrock/count_tokens/mantle_handler.py | 3 +- .../bedrock/count_tokens/transformation.py | 5 +- litellm/llms/bedrock/embed/embedding.py | 7 +- litellm/llms/bedrock/image_edit/handler.py | 3 +- .../bedrock/image_generation/image_handler.py | 3 +- litellm/llms/bedrock/rerank/handler.py | 3 +- .../llms/bedrock/responses/transformation.py | 6 +- .../bedrock/vector_stores/transformation.py | 6 +- .../bedrock_mantle/messages/transformation.py | 3 +- .../responses/transformation.py | 3 +- .../image_edit/transformation.py | 7 +- .../image_generation/transformation.py | 6 +- .../llms/chatgpt/responses/transformation.py | 4 +- .../llms/cloudflare/chat/transformation.py | 3 +- litellm/llms/cloudflare/common_utils.py | 5 +- litellm/llms/cohere/rerank/transformation.py | 7 +- .../llms/cohere/rerank_v2/transformation.py | 7 +- litellm/llms/cometapi/chat/transformation.py | 11 +- litellm/llms/cometapi/embed/transformation.py | 5 +- .../image_generation/transformation.py | 8 +- litellm/llms/custom_httpx/llm_http_handler.py | 6 +- litellm/llms/dashscope/chat/transformation.py | 6 +- .../llms/dashscope/embed/transformation.py | 6 +- .../llms/dashscope/rerank/transformation.py | 5 +- .../llms/dashscope/videos/transformation.py | 6 +- .../llms/databricks/chat/transformation.py | 6 +- litellm/llms/databricks/common_utils.py | 12 +- .../databricks/decisions/transformation.py | 5 +- .../databricks/responses/transformation.py | 4 +- .../audio_transcription/transformation.py | 4 +- litellm/llms/deepgram/common_utils.py | 3 +- litellm/llms/deepseek/chat/transformation.py | 8 +- .../llms/deepseek/messages/transformation.py | 5 +- .../chat/transformation.py | 6 +- .../llms/duckduckgo/search/transformation.py | 3 +- litellm/llms/e2b/sandbox/transformation.py | 7 +- litellm/llms/edenai/chat/transformation.py | 4 +- litellm/llms/edenai/common_utils.py | 3 +- .../audio_transcription/transformation.py | 8 +- .../text_to_speech/transformation.py | 7 +- litellm/llms/exa_ai/search/transformation.py | 7 +- .../flux_lora_depth_transformation.py | 5 +- .../fal_ai/image_generation/transformation.py | 11 +- litellm/llms/fal_ai/videos/transformation.py | 6 +- litellm/llms/fastcrw/search/transformation.py | 7 +- .../llms/firecrawl/search/transformation.py | 7 +- .../llms/fireworks_ai/chat/transformation.py | 3 +- .../fireworks_ai/rerank/transformation.py | 5 +- .../fireworks_ai/responses/transformation.py | 5 +- litellm/llms/gdc/chat/transformation.py | 3 +- litellm/llms/gemini/agents/transformation.py | 13 +- .../audio_transcription/transformation.py | 4 +- litellm/llms/gemini/common_utils.py | 7 +- litellm/llms/gemini/count_tokens/handler.py | 3 +- litellm/llms/gemini/files/transformation.py | 14 +- .../llms/gemini/image_edit/transformation.py | 6 +- .../gemini/image_generation/transformation.py | 7 +- .../gemini/interactions/transformation.py | 14 +- .../llms/gemini/realtime/transformation.py | 6 +- .../gemini/vector_stores/transformation.py | 15 +- litellm/llms/gemini/videos/transformation.py | 7 +- litellm/llms/gigachat/chat/transformation.py | 4 +- .../llms/gigachat/embedding/transformation.py | 4 +- litellm/llms/gigachat/file_handler.py | 7 +- .../gigachat/passthrough/transformation.py | 5 +- litellm/llms/gigachat/utils.py | 2 +- .../embedding/transformation.py | 9 +- .../github_copilot/messages/transformation.py | 5 +- .../responses/transformation.py | 9 +- .../llms/gradient_ai/chat/transformation.py | 5 +- litellm/llms/heroku/chat/transformation.py | 10 +- .../hosted_vllm/embedding/transformation.py | 10 +- .../hosted_vllm/image_edit/transformation.py | 5 +- .../llms/hosted_vllm/rerank/transformation.py | 3 +- .../hosted_vllm/responses/transformation.py | 5 +- .../transcriptions/transformation.py | 7 +- .../llms/hosted_vllm/videos/transformation.py | 6 +- .../llms/huggingface/chat/transformation.py | 5 +- .../llms/huggingface/rerank/transformation.py | 11 +- .../llms/infinity/embedding/transformation.py | 7 +- .../llms/infinity/rerank/transformation.py | 7 +- .../llms/jina_ai/embedding/transformation.py | 8 +- litellm/llms/langflow/chat/transformation.py | 4 +- litellm/llms/langgraph/chat/transformation.py | 8 +- litellm/llms/lemonade/chat/transformation.py | 5 +- litellm/llms/linkup/search/transformation.py | 7 +- .../image_edit/transformation.py | 4 +- .../image_generation/transformation.py | 4 +- .../litellm_proxy/responses/transformation.py | 6 +- litellm/llms/manus/files/transformation.py | 8 +- .../llms/manus/responses/transformation.py | 6 +- .../milvus/vector_stores/transformation.py | 3 +- litellm/llms/minimax/chat/transformation.py | 12 +- .../llms/minimax/messages/transformation.py | 16 +- .../minimax/text_to_speech/transformation.py | 10 +- .../mistral/audio_speech/transformation.py | 5 +- .../audio_transcription/transformation.py | 4 +- .../llms/mistral/batches/transformation.py | 9 +- litellm/llms/mistral/chat/transformation.py | 4 +- litellm/llms/mistral/files/transformation.py | 8 +- .../llms/modelscope/chat/transformation.py | 8 +- .../image_generation/transformation.py | 6 +- .../mongodb/vector_stores/transformation.py | 3 +- litellm/llms/moonshot/chat/transformation.py | 8 +- litellm/llms/nimble/search/transformation.py | 6 +- .../rerank/ranking_transformation.py | 3 +- .../llms/nvidia_nim/rerank/transformation.py | 3 +- litellm/llms/oci/chat/transformation.py | 3 +- litellm/llms/oci/embed/transformation.py | 3 +- litellm/llms/ollama/chat/transformation.py | 6 +- litellm/llms/ollama/common_utils.py | 5 +- litellm/llms/ollama/completion/handler.py | 12 +- .../llms/ollama/completion/transformation.py | 6 +- litellm/llms/oobabooga/chat/oobabooga.py | 5 +- .../llms/openai/chat/gpt_transformation.py | 3 +- .../llms/openai/containers/transformation.py | 4 +- litellm/llms/openai/evals/transformation.py | 20 +- .../llms/openai/image_edit/transformation.py | 6 +- .../openai/realtime/http_transformation.py | 7 +- .../responses/count_tokens/transformation.py | 6 +- .../llms/openai/responses/transformation.py | 11 +- .../transcriptions/whisper_transformation.py | 5 +- .../vector_store_files/transformation.py | 9 +- .../openai/vector_stores/transformation.py | 9 +- litellm/llms/openai/videos/transformation.py | 16 +- litellm/llms/openai_like/common_utils.py | 6 +- litellm/llms/openai_like/dynamic_config.py | 19 +- .../openai_like/messages/transformation.py | 3 +- .../openai_like/responses/transformation.py | 4 +- .../openrouter/embedding/transformation.py | 11 +- .../openrouter/image_edit/transformation.py | 8 +- .../image_generation/transformation.py | 6 +- .../openrouter/responses/transformation.py | 5 +- .../opensandbox/sandbox/transformation.py | 9 +- .../audio_transcription/transformation.py | 6 +- litellm/llms/ovhcloud/chat/transformation.py | 6 +- .../llms/ovhcloud/embedding/transformation.py | 6 +- .../perplexity/embedding/transformation.py | 3 +- .../perplexity/responses/transformation.py | 3 +- .../llms/perplexity/search/transformation.py | 7 +- .../pg_vector/vector_stores/transformation.py | 9 +- litellm/llms/predibase/chat/transformation.py | 7 +- litellm/llms/ragflow/chat/transformation.py | 4 +- .../ragflow/vector_stores/transformation.py | 14 +- .../llms/recraft/image_edit/transformation.py | 8 +- .../image_generation/transformation.py | 8 +- .../image_generation/transformation.py | 15 +- .../runwayml/text_to_speech/transformation.py | 13 +- .../llms/runwayml/videos/transformation.py | 10 +- .../vector_stores/transformation.py | 3 +- .../sambanova/embedding/transformation.py | 7 +- litellm/llms/sap/chat/transformation.py | 7 +- litellm/llms/sap/credentials.py | 3 +- litellm/llms/sap/embed/transformation.py | 7 +- litellm/llms/scaledown/chat/transformation.py | 5 +- .../audio_transcription/transformation.py | 4 +- .../llms/scaleway/rerank/transformation.py | 5 +- litellm/llms/searxng/search/transformation.py | 8 +- litellm/llms/serper/search/transformation.py | 8 +- litellm/llms/snowflake/chat/transformation.py | 5 +- .../snowflake/embedding/transformation.py | 3 +- litellm/llms/snowflake/utils.py | 8 +- .../soniox/audio_transcription/handler.py | 27 +- .../image_generation/transformation.py | 6 +- litellm/llms/tavily/search/transformation.py | 7 +- litellm/llms/tencent/chat/transformation.py | 6 +- .../llms/tencent/messages/transformation.py | 3 +- .../topaz/image_variations/transformation.py | 4 +- .../vercel_ai_gateway/chat/transformation.py | 3 +- .../embedding/transformation.py | 9 +- .../gemini_transcribe_transformation.py | 5 +- .../audio_transcription/transformation.py | 5 +- litellm/llms/vertex_ai/batches/handler.py | 3 +- .../llms/vertex_ai/files/transformation.py | 5 +- .../vertex_imagen_transformation.py | 11 +- .../vertex_ai/interactions/transformation.py | 8 +- .../llms/vertex_ai/realtime/transformation.py | 6 +- .../text_to_speech/transformation.py | 11 +- .../vector_stores/rag_api/transformation.py | 3 +- .../count_tokens/handler.py | 13 +- litellm/llms/vertex_ai/vertex_llm_base.py | 43 +- .../llms/vertex_ai/videos/transformation.py | 14 +- .../volcengine/embedding/transformation.py | 9 +- .../volcengine/responses/transformation.py | 10 +- .../llms/voyage/embedding/transformation.py | 8 +- .../embedding/transformation_contextual.py | 8 +- .../embedding/transformation_multimodal.py | 8 +- litellm/llms/voyage/rerank/transformation.py | 5 +- .../audio_transcription/transformation.py | 4 +- litellm/llms/watsonx/chat/transformation.py | 3 +- .../llms/watsonx/completion/transformation.py | 3 +- litellm/llms/watsonx/embed/transformation.py | 3 +- litellm/llms/watsonx/rerank/transformation.py | 3 +- .../xai/audio_transcription/transformation.py | 3 +- litellm/llms/xai/batches/transformation.py | 9 +- litellm/llms/xai/common_utils.py | 3 +- litellm/llms/xai/files/transformation.py | 10 +- litellm/llms/xai/responses/transformation.py | 6 +- litellm/llms/you_com/search/transformation.py | 3 +- litellm/main.py | 24 +- litellm/rag/ingestion/gemini_ingestion.py | 7 +- .../providers/_api_base_url_join_support.py | 163 ++++++++ .../providers/test_api_base_url_join_chaos.py | 176 +++++++++ ...test_api_base_url_join_host_owned_proxy.py | 140 +++++++ .../providers/test_api_base_url_join_wire.py | 373 ++++++++++++++++++ .../unit/litellm_core_utils/test_url_utils.py | 81 ++++ .../realtime/test_azure_realtime_handler.py | 12 + .../llms/azure/test_azure_common_utils.py | 18 + .../unit/llms/azure_ai/agents/test_handler.py | 23 +- ...e_anthropic_count_tokens_transformation.py | 22 +- ...azure_anthropic_messages_transformation.py | 11 + .../test_agentcore_transformation.py | 14 + .../evals/test_openai_evals_transformation.py | 11 + .../test_api_base_trailing_slash_routes.py | 119 ++++++ .../test_get_complete_url_trailing_slash.py | 278 +++++++++++++ .../llms/test_predibase_transformation.py | 14 + .../llms/vertex_ai/test_vertex_llm_base.py | 16 + 256 files changed, 2260 insertions(+), 922 deletions(-) create mode 100644 tests/integration/providers/_api_base_url_join_support.py create mode 100644 tests/integration/providers/test_api_base_url_join_chaos.py create mode 100644 tests/integration/providers/test_api_base_url_join_host_owned_proxy.py create mode 100644 tests/integration/providers/test_api_base_url_join_wire.py create mode 100644 tests/unit/llms/test_api_base_trailing_slash_routes.py create mode 100644 tests/unit/llms/test_get_complete_url_trailing_slash.py diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index c593acbe8fb..2276efc4e34 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -216,10 +216,12 @@ jobs: - shard: llms-providers test-path: |- + tests/unit/llms/test_api_base_trailing_slash_routes.py tests/unit/llms/test_cache_control_and_reasoning.py tests/unit/llms/test_custom_llm.py tests/unit/llms/test_file_content_block.py tests/unit/llms/test_file_search_responses.py + tests/unit/llms/test_get_complete_url_trailing_slash.py tests/unit/llms/test_lifecycle_fix.py tests/unit/llms/test_oss_decision.py tests/unit/llms/test_polling_url_origin_match.py diff --git a/litellm/litellm_core_utils/url_utils.py b/litellm/litellm_core_utils/url_utils.py index 43e16599bf6..4a1eaa87d3a 100644 --- a/litellm/litellm_core_utils/url_utils.py +++ b/litellm/litellm_core_utils/url_utils.py @@ -23,7 +23,7 @@ import asyncio import socket from ipaddress import ip_address, ip_network from typing import Any, Final, Protocol -from urllib.parse import quote, urlparse, urlunparse +from urllib.parse import quote, urlparse, urlsplit, urlunparse import httpx from typing_extensions import ReadOnly, TypedDict @@ -129,6 +129,28 @@ def encode_url_path_segments(value: object, *, field_name: str = "path") -> str: return "/".join(encode_url_path_segment(segment, field_name=field_name) for segment in value_str.split("/")) +def _strip_trailing_slashes_keeping_the_scheme(api_base: str) -> str: + stripped: Final = api_base.rstrip("/") + parts: Final = urlsplit(stripped) + if parts.scheme and not (parts.netloc or parts.path or parts.query or parts.fragment): + return api_base + if not parts.query or not parts.path.endswith("/"): + return stripped + url_before_query, _, query = stripped.partition("?") + return f"{url_before_query.rstrip('/')}?{query}" + + +def join_api_base(api_base: str, route: str) -> str: + return f"{_strip_trailing_slashes_keeping_the_scheme(api_base)}/{route.lstrip('/')}" + + +def ensure_api_route(api_base: str, route: str) -> str: + base: Final = _strip_trailing_slashes_keeping_the_scheme(api_base) + if urlsplit(base).path.endswith(f"/{route.lstrip('/')}"): + return base + return join_api_base(base, route) + + def _is_blocked_ip(addr: str) -> bool: """Return True for any IP not safe to reach from a user-supplied URL. diff --git a/litellm/llms/aiml/image_generation/transformation.py b/litellm/llms/aiml/image_generation/transformation.py index 931038689cc..bb981983e98 100644 --- a/litellm/llms/aiml/image_generation/transformation.py +++ b/litellm/llms/aiml/image_generation/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -121,7 +122,7 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig): complete_url = complete_url.rstrip("/") # Strip /v1 suffix if present since IMAGE_GENERATION_ENDPOINT already includes v1 complete_url = complete_url.removesuffix("/v1") - complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" + complete_url = join_api_base(complete_url, self.IMAGE_GENERATION_ENDPOINT) return complete_url def validate_environment( diff --git a/litellm/llms/aiohttp_openai/chat/transformation.py b/litellm/llms/aiohttp_openai/chat/transformation.py index dba7a7405ab..74f782b56e7 100644 --- a/litellm/llms/aiohttp_openai/chat/transformation.py +++ b/litellm/llms/aiohttp_openai/chat/transformation.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Any, Final from aiohttp import ClientResponse from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, ModelResponse @@ -44,12 +45,7 @@ class AiohttpOpenAIChatConfig(OpenAILikeChatConfig): Ensure - /v1/chat/completions is at the end of the url """ - if api_base is None: - api_base = "https://api.openai.com" - - if not api_base.endswith("/chat/completions"): - api_base += "/chat/completions" - return api_base + return ensure_api_route("https://api.openai.com" if api_base is None else api_base, "chat/completions") def validate_environment( self, diff --git a/litellm/llms/anthropic/batches/transformation.py b/litellm/llms/anthropic/batches/transformation.py index b95b734c761..da6c2871289 100644 --- a/litellm/llms/anthropic/batches/transformation.py +++ b/litellm/llms/anthropic/batches/transformation.py @@ -7,7 +7,7 @@ import httpx from httpx import Headers, Response from typing_extensions import ReadOnly, TypedDict -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, ensure_api_route, join_api_base from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest @@ -106,10 +106,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig): data: CreateBatchRequest, ) -> str: """Get the complete URL for batch creation request.""" - api_base = api_base or self.anthropic_model_info.get_api_base(api_base) - if not api_base.endswith("/v1/messages/batches"): - api_base = f"{api_base.rstrip('/')}/v1/messages/batches" - return api_base + api_base = self.anthropic_model_info.resolve_anthropic_api_base(api_base) + return ensure_api_route(api_base, "v1/messages/batches") def transform_create_batch_request( self, @@ -158,9 +156,9 @@ class AnthropicBatchesConfig(BaseBatchesConfig): Returns: Complete URL for Anthropic batch retrieval: {api_base}/v1/messages/batches/{batch_id} """ - api_base = api_base or self.anthropic_model_info.get_api_base(api_base) + api_base = self.anthropic_model_info.resolve_anthropic_api_base(api_base) encoded_batch_id: Final = encode_url_path_segment(batch_id, field_name="batch_id") - return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}" + return join_api_base(api_base, f"v1/messages/batches/{encoded_batch_id}") def transform_retrieve_batch_request( self, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 6f08d6b0d61..1f5e4e7de39 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -35,6 +35,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import parts_of, ) from litellm.litellm_core_utils.reasoning_effort_utils import non_numeric_thinking_budget_error +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.wif import ( aget_anthropic_wif_token, anthropic_base_without_chat_suffix, @@ -468,7 +469,7 @@ def _fetch_anthropic_models_page( # which a read-only one cannot support query: Final = f"?after_id={quote(after_id)}" if after_id else "" response: Final = litellm.module_level_client.get( - url=f"{api_base}/v1/models{query}", + url=f"{join_api_base(api_base, 'v1/models')}{query}", headers=headers, follow_redirects=False, ) @@ -1406,7 +1407,7 @@ class AnthropicModelInfo(BaseLLMModelInfo): return {**caller_headers, **anthropic_headers} @staticmethod - def get_api_base(api_base: str | None = None) -> str | None: + def resolve_anthropic_api_base(api_base: str | None = None) -> str: from litellm.secret_managers.main import get_secret_str return ( @@ -1416,6 +1417,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): or "https://api.anthropic.com" ) + @staticmethod + def get_api_base(api_base: str | None = None) -> str | None: + return AnthropicModelInfo.resolve_anthropic_api_base(api_base) + @staticmethod def get_api_key(api_key: str | None = None) -> str | None: """An empty or whitespace-only key counts as unset: it can never authenticate anything, and @@ -1533,14 +1538,14 @@ class AnthropicModelInfo(BaseLLMModelInfo): api_base: str | None, litellm_params: Mapping[str, object] | None, ) -> list[str]: # mutable-ok: matches get_models' list[str] contract shared by every provider override - resolved_api_base: Final = AnthropicModelInfo.get_api_base(api_base) + resolved_api_base: Final = AnthropicModelInfo.resolve_anthropic_api_base(api_base) auth_header: Final = AnthropicModelInfo.get_auth_header( api_key, resolved_api_base, litellm_params=litellm_params, allow_workload_identity=config_allows_workload_identity(self), ) - if resolved_api_base is None or auth_header is None: + if auth_header is None: raise ValueError( "ANTHROPIC_API_BASE/ANTHROPIC_BASE_URL or ANTHROPIC_API_KEY/ANTHROPIC_AUTH_TOKEN (or workload " "identity federation via ANTHROPIC_FEDERATION_RULE_ID/ANTHROPIC_ORGANIZATION_ID/" diff --git a/litellm/llms/anthropic/count_tokens/transformation.py b/litellm/llms/anthropic/count_tokens/transformation.py index 592b5fb0f00..c2d87c641be 100644 --- a/litellm/llms/anthropic/count_tokens/transformation.py +++ b/litellm/llms/anthropic/count_tokens/transformation.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import anthropic_system_blocks, split_leading_system_run, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers from litellm.llms.anthropic.wif import resolve_anthropic_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionImageObject, ValidUserMessageContentTypes @@ -317,7 +318,7 @@ class AnthropicCountTokensConfig: Returns: The endpoint URL for the CountTokens API """ - return resolve_anthropic_base(api_base) + "/v1/messages/count_tokens" + return join_api_base(resolve_anthropic_base(api_base), "v1/messages/count_tokens") def transform_request_to_count_tokens( self, diff --git a/litellm/llms/anthropic/files/handler.py b/litellm/llms/anthropic/files/handler.py index f1f9c204b58..1a793e32317 100644 --- a/litellm/llms/anthropic/files/handler.py +++ b/litellm/llms/anthropic/files/handler.py @@ -10,7 +10,7 @@ import litellm from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.types.llms.openai import ( FileContentRequest, @@ -74,7 +74,7 @@ class AnthropicFilesHandler: batch_id = file_id # Get Anthropic API credentials - api_base = self.anthropic_model_info.get_api_base(api_base) + api_base = self.anthropic_model_info.resolve_anthropic_api_base(api_base) auth_header: Final = await self.anthropic_model_info.aget_auth_header( api_key, api_base, litellm_params=litellm_params, allow_workload_identity=True ) @@ -84,7 +84,7 @@ class AnthropicFilesHandler: # Construct the Anthropic batch results URL encoded_batch_id: Final = encode_url_path_segment(batch_id, field_name="batch_id") - results_url: Final = f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}/results" + results_url: Final = join_api_base(api_base, f"v1/messages/batches/{encoded_batch_id}/results") # Prepare headers headers: Final = { diff --git a/litellm/llms/anthropic/files/transformation.py b/litellm/llms/anthropic/files/transformation.py index 8433235dde4..db14d3366ff 100644 --- a/litellm/llms/anthropic/files/transformation.py +++ b/litellm/llms/anthropic/files/transformation.py @@ -22,7 +22,7 @@ from openai.types.file_deleted import FileDeleted from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, @@ -80,7 +80,7 @@ class AnthropicFilesConfig(BaseFilesConfig): stream: bool | None = None, ) -> str: api_base = AnthropicModelInfo.get_api_base(api_base) or ANTHROPIC_FILES_API_BASE - return f"{api_base.rstrip('/')}/v1/files" + return join_api_base(api_base, "v1/files") def get_error_class( self, @@ -226,7 +226,7 @@ class AnthropicFilesConfig(BaseFilesConfig): ) -> tuple[str, dict]: api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id") - return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {} + return join_api_base(api_base, f"v1/files/{encoded_file_id}"), {} def transform_retrieve_file_response( self, @@ -245,7 +245,7 @@ class AnthropicFilesConfig(BaseFilesConfig): ) -> tuple[str, dict]: api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id") - return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {} + return join_api_base(api_base, f"v1/files/{encoded_file_id}"), {} def transform_delete_file_response( self, @@ -264,7 +264,7 @@ class AnthropicFilesConfig(BaseFilesConfig): litellm_params: dict, ) -> tuple[str, dict]: api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE - url: Final = f"{api_base.rstrip('/')}/v1/files" + url: Final = join_api_base(api_base, "v1/files") params: Final[dict[str, str]] = {} if purpose: params["purpose"] = purpose @@ -299,8 +299,8 @@ class AnthropicFilesConfig(BaseFilesConfig): api_base: Final = AnthropicModelInfo.get_api_base(litellm_params.get("api_base")) or ANTHROPIC_FILES_API_BASE encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id") if file_id.startswith(ANTHROPIC_MESSAGE_BATCH_ID_PREFIX): - return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_file_id}/results", {} - return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {} + return join_api_base(api_base, f"v1/messages/batches/{encoded_file_id}/results"), {} + return join_api_base(api_base, f"v1/files/{encoded_file_id}/content"), {} def transform_file_content_response( self, diff --git a/litellm/llms/anthropic/pass_through/messages/transformation.py b/litellm/llms/anthropic/pass_through/messages/transformation.py index a71cd72b768..9f3aa0130d1 100644 --- a/litellm/llms/anthropic/pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/pass_through/messages/transformation.py @@ -9,6 +9,7 @@ from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_b from litellm.exceptions import AuthenticationError from litellm.litellm_core_utils.core_helpers import drops_unsupported_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards from litellm.llms.base_llm.anthropic_messages.transformation import ( @@ -287,10 +288,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - api_base = AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com" - if not api_base.endswith("/v1/messages"): - api_base = f"{api_base}/v1/messages" - return api_base + return ensure_api_route(AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com", "v1/messages") def validate_anthropic_messages_environment( self, diff --git a/litellm/llms/anthropic/skills/transformation.py b/litellm/llms/anthropic/skills/transformation.py index 6d419a56889..8ea9d192c24 100644 --- a/litellm/llms/anthropic/skills/transformation.py +++ b/litellm/llms/anthropic/skills/transformation.py @@ -9,7 +9,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.skills.transformation import ( BaseSkillsAPIConfig, LiteLLMLoggingObj, @@ -75,13 +75,11 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig): """Get complete URL for Anthropic Skills API""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo - if api_base is None: - api_base = AnthropicModelInfo.get_api_base() - + resolved_api_base: Final = AnthropicModelInfo.get_api_base(api_base) or "https://api.anthropic.com" if skill_id: encoded_skill_id: Final = encode_url_path_segment(skill_id, field_name="skill_id") - return f"{api_base}/v1/skills/{encoded_skill_id}" - return f"{api_base}/v1/{endpoint}" + return join_api_base(resolved_api_base, f"v1/skills/{encoded_skill_id}") + return join_api_base(resolved_api_base, f"v1/{endpoint}") def transform_create_skill_request( self, diff --git a/litellm/llms/anthropic/wif.py b/litellm/llms/anthropic/wif.py index b47c4f28eb2..ccb81c1244f 100644 --- a/litellm/llms/anthropic/wif.py +++ b/litellm/llms/anthropic/wif.py @@ -15,6 +15,7 @@ from typing_extensions import assert_never import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.auth.client_credentials import keycloak_assertion_source from litellm.llms.base_llm.auth.identity_source import ( AnthropicIdentitySourceKind, @@ -324,7 +325,7 @@ def anthropic_internal_issuer_jwks(credential_values: Mapping[str, object]) -> A def build_anthropic_wif_spec(params: AnthropicWifParams, api_base: str) -> TokenExchangeSpec: return TokenExchangeSpec( - token_url=api_base.rstrip("/") + ANTHROPIC_TOKEN_EXCHANGE_PATH, + token_url=join_api_base(api_base, ANTHROPIC_TOKEN_EXCHANGE_PATH), assertion_ref=params.assertion_ref, assertion_field="assertion", static_body=MappingProxyType( diff --git a/litellm/llms/apiserpent/search/transformation.py b/litellm/llms/apiserpent/search/transformation.py index bb8abff7272..faf1903ffe8 100644 --- a/litellm/llms/apiserpent/search/transformation.py +++ b/litellm/llms/apiserpent/search/transformation.py @@ -14,6 +14,7 @@ from urllib.parse import urlencode import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.apiserpent.search.defaults import ( DEEP_SEARCH_PATH, NUM_MAX, @@ -77,13 +78,11 @@ class APISerpentSearchConfig(BaseSearchConfig): Build the search URL. APISerpent uses GET, so the transformed request is serialized into the query string. The endpoint path (quick vs deep) is always applied; an ``api_base`` / ``APISERPENT_API_BASE`` override only - changes the host. The ``endswith`` guard keeps this idempotent, since the + changes the host. ``ensure_api_route`` keeps this idempotent, since the handler re-invokes this method with the already-resolved URL as api_base. """ - base = (api_base or get_secret_str("APISERPENT_API_BASE") or APISERPENT_BASE).rstrip("/") path: Final = DEEP_SEARCH_PATH if self._is_deep_search(optional_params) else QUICK_SEARCH_PATH - if not base.endswith(path): - base = f"{base}{path}" + base: Final = ensure_api_route(api_base or get_secret_str("APISERPENT_API_BASE") or APISERPENT_BASE, path) if data and isinstance(data, dict) and APISERPENT_PARAMS_KEY in data: query_string: Final = urlencode(data[APISERPENT_PARAMS_KEY], doseq=True) diff --git a/litellm/llms/azure/audio_transcription/transformation.py b/litellm/llms/azure/audio_transcription/transformation.py index 858f122f65a..a5fde7f7158 100644 --- a/litellm/llms/azure/audio_transcription/transformation.py +++ b/litellm/llms/azure/audio_transcription/transformation.py @@ -13,6 +13,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -110,7 +111,7 @@ class AzureSpeechAudioTranscriptionConfig(BaseAudioTranscriptionConfig): "language": optional_params.get("language", self.DEFAULT_LANGUAGE), "format": self._get_azure_response_format(optional_params.get("response_format")), } - return f"{base_url}{self.STT_ENDPOINT_PATH}?{urlencode(query_params)}" + return f"{join_api_base(base_url, self.STT_ENDPOINT_PATH)}?{urlencode(query_params)}" def transform_audio_transcription_request( self, @@ -153,7 +154,6 @@ class AzureSpeechAudioTranscriptionConfig(BaseAudioTranscriptionConfig): ) def _resolve_stt_base_url(self, api_base: str) -> str: - api_base = api_base.rstrip("/") parsed_url: Final = urlparse(api_base) hostname: Final = parsed_url.hostname or "" diff --git a/litellm/llms/azure/common_utils.py b/litellm/llms/azure/common_utils.py index 52e3cdf380e..23c5aa35f4b 100644 --- a/litellm/llms/azure/common_utils.py +++ b/litellm/llms/azure/common_utils.py @@ -18,6 +18,7 @@ import litellm from litellm._logging import verbose_logger from litellm.caching.caching import DualCache from litellm.constants import DEFAULT_MAX_RETRIES +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient @@ -616,7 +617,7 @@ class BaseAzureLLM(BaseOpenAILLM): v1_params: Final[_AzureV1ClientParams] = { "api_key": v1_api_key, - "base_url": f"{api_base}/openai/v1/", + "base_url": f"{ensure_api_route(api_base or '', 'openai/v1')}/", } if "timeout" in azure_client_params: v1_params["timeout"] = azure_client_params["timeout"] diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 75f96bfdf63..3d4576fd7ac 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -10,6 +10,7 @@ from typing import Final, Protocol, cast from litellm._logging import verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.types.realtime import RealtimeQueryParams from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -136,7 +137,8 @@ class AzureOpenAIRealtime(AzureChatCompletion): query_parts.append(urlencode({"intent": intent})) qs: Final = "&".join(query_parts) - return f"{api_base}{path}?{qs}" if qs else f"{api_base}{path}" + endpoint_url: Final = join_api_base(api_base, path) + return f"{endpoint_url}?{qs}" if qs else endpoint_url def construct_url( self, diff --git a/litellm/llms/azure/realtime/http_transformation.py b/litellm/llms/azure/realtime/http_transformation.py index 542a6a8c02f..545c3348182 100644 --- a/litellm/llms/azure/realtime/http_transformation.py +++ b/litellm/llms/azure/realtime/http_transformation.py @@ -3,6 +3,7 @@ from typing import Final import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str @@ -39,9 +40,9 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig): return _azure_realtime_v1_url(self.get_api_base(api_base), "calls") def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: - base: Final = self.get_api_base(api_base).rstrip("/") + base: Final = self.get_api_base(api_base) version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17" - return f"{base}/openai/realtime/transcription_sessions?api-version={version}" + return f"{join_api_base(base, 'openai/realtime/transcription_sessions')}?api-version={version}" def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]: return { diff --git a/litellm/llms/azure/search/transformation.py b/litellm/llms/azure/search/transformation.py index f2bb2ecc68c..f7cdf88ade0 100644 --- a/litellm/llms/azure/search/transformation.py +++ b/litellm/llms/azure/search/transformation.py @@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Final, Literal import httpx from pydantic import ConfigDict, ValidationError +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.search.transformation import ( @@ -340,10 +341,7 @@ class BingGroundingSearchConfig(BaseSearchConfig): f"{PROJECT_ENDPOINT_ENV} is not set. Set it to your Microsoft Foundry project " f"endpoint, e.g. https://.services.ai.azure.com/api/projects/." ) - trimmed: Final = resolved_base.rstrip("/") - if trimmed.endswith(_RESPONSES_PATH): - return trimmed - return f"{trimmed}{_RESPONSES_PATH}" + return ensure_api_route(resolved_base, _RESPONSES_PATH) def transform_search_request( self, diff --git a/litellm/llms/azure/text_to_speech/transformation.py b/litellm/llms/azure/text_to_speech/transformation.py index eed7a3178ca..acae650ed75 100644 --- a/litellm/llms/azure/text_to_speech/transformation.py +++ b/litellm/llms/azure/text_to_speech/transformation.py @@ -11,6 +11,7 @@ from urllib.parse import urlparse import httpx import litellm +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, @@ -329,8 +330,6 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): f"https://{{region}}.{self.TTS_SPEECH_DOMAIN}" ) - # Remove trailing slash and parse URL - api_base = api_base.rstrip("/") parsed_url: Final = urlparse(api_base) hostname: Final = parsed_url.hostname or "" @@ -339,14 +338,10 @@ class AzureAVATextToSpeechConfig(BaseTextToSpeechConfig): region: Final = self._extract_region_from_hostname(hostname=hostname, domain=self.COGNITIVE_SERVICES_DOMAIN) return self._build_tts_url(region=region) - # Check if it's already a TTS endpoint if self._is_tts_endpoint(hostname=hostname): - if not api_base.endswith(self.TTS_ENDPOINT_PATH): - return f"{api_base}{self.TTS_ENDPOINT_PATH}" - return api_base + return ensure_api_route(api_base, self.TTS_ENDPOINT_PATH) - # Assume it's a custom endpoint, append the path - return f"{api_base}{self.TTS_ENDPOINT_PATH}" + return join_api_base(api_base, self.TTS_ENDPOINT_PATH) def _is_cognitive_services_endpoint(self, hostname: str) -> bool: """Check if hostname is a Cognitive Services endpoint""" diff --git a/litellm/llms/azure_ai/agents/handler.py b/litellm/llms/azure_ai/agents/handler.py index 0b0548dd3a2..94f1399f01b 100644 --- a/litellm/llms/azure_ai/agents/handler.py +++ b/litellm/llms/azure_ai/agents/handler.py @@ -30,7 +30,7 @@ from typing_extensions import ReadOnly from litellm._logging import verbose_logger from litellm.litellm_core_utils.hidden_params import set_hidden_param -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.azure_ai.agents.transformation import ( AzureAIAgentsConfig, AzureAIAgentsError, @@ -130,28 +130,29 @@ class AzureAIAgentsHandler: # See: https://learn.microsoft.com/en-us/azure/ai-foundry/agents/quickstart # ------------------------------------------------------------------------- def _build_thread_url(self, api_base: str, api_version: str) -> str: - return f"{api_base}/threads?api-version={api_version}" + return f"{join_api_base(api_base, 'threads')}?api-version={api_version}" def _build_messages_url(self, api_base: str, thread_id: str, api_version: str) -> str: encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id") - return f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}" + return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/messages')}?api-version={api_version}" def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str: encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id") - return f"{api_base}/threads/{encoded_thread_id}/runs?api-version={api_version}" + return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/runs')}?api-version={api_version}" def _build_run_status_url(self, api_base: str, thread_id: str, run_id: str, api_version: str) -> str: encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id") encoded_run_id: Final = encode_url_path_segment(run_id, field_name="run_id") - return f"{api_base}/threads/{encoded_thread_id}/runs/{encoded_run_id}?api-version={api_version}" + run_status_route: Final = f"threads/{encoded_thread_id}/runs/{encoded_run_id}" + return f"{join_api_base(api_base, run_status_route)}?api-version={api_version}" def _build_list_messages_url(self, api_base: str, thread_id: str, api_version: str) -> str: encoded_thread_id: Final = encode_url_path_segment(thread_id, field_name="thread_id") - return f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}" + return f"{join_api_base(api_base, f'threads/{encoded_thread_id}/messages')}?api-version={api_version}" def _build_create_thread_and_run_url(self, api_base: str, api_version: str) -> str: """URL for the create-thread-and-run endpoint (supports streaming).""" - return f"{api_base}/threads/runs?api-version={api_version}" + return f"{join_api_base(api_base, 'threads/runs')}?api-version={api_version}" # ------------------------------------------------------------------------- # Response Helpers @@ -281,8 +282,6 @@ class AzureAIAgentsHandler: api_version: Final = optional_params.get("api_version", self.config.DEFAULT_API_VERSION) agent_id: Final = self.config.get_agent_id(model, optional_params) thread_id: Final = optional_params.get("thread_id") - api_base = api_base.rstrip("/") - verbose_logger.debug("Azure AI Agents completion - api_base: %s, agent_id: %s", api_base, agent_id) return headers, api_version, agent_id, thread_id, api_base diff --git a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py index 5946a1f4480..6e3cfce5eb4 100644 --- a/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py +++ b/litellm/llms/azure_ai/anthropic/count_tokens/transformation.py @@ -5,8 +5,10 @@ Extends the base Anthropic CountTokens transformation with Azure authentication. """ from typing import Final +from urllib.parse import urlsplit from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) @@ -76,14 +78,8 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig): Returns: The endpoint URL for the CountTokens API """ - # Azure AI Anthropic endpoint format: - # https://.services.ai.azure.com/anthropic/v1/messages/count_tokens - api_base = api_base.rstrip("/") - - # Ensure the URL has /anthropic path - if not api_base.endswith("/anthropic"): - if "/anthropic" not in api_base: - api_base = f"{api_base}/anthropic" - - # Add the count_tokens path - return f"{api_base}/v1/messages/count_tokens" + base: Final = api_base.rstrip("/") + if base.endswith("/v1/messages"): + return join_api_base(base, "count_tokens") + anthropic_base: Final = base if "/anthropic" in urlsplit(base).path else join_api_base(base, "anthropic") + return join_api_base(anthropic_base, "v1/messages/count_tokens") diff --git a/litellm/llms/azure_ai/anthropic/messages_transformation.py b/litellm/llms/azure_ai/anthropic/messages_transformation.py index 91c75661688..2aacc5771d0 100644 --- a/litellm/llms/azure_ai/anthropic/messages_transformation.py +++ b/litellm/llms/azure_ai/anthropic/messages_transformation.py @@ -9,6 +9,7 @@ from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) from litellm.llms.azure.common_utils import BaseAzureLLM +from litellm.llms.azure_ai.anthropic.transformation import azure_anthropic_messages_url from litellm.types.router import GenericLiteLLMParams @@ -96,25 +97,7 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig): "Expected format: https://.services.ai.azure.com/anthropic" ) - # Ensure the URL ends with /v1/messages - api_base = api_base.rstrip("/") - if api_base.endswith("/v1/messages") or api_base.endswith("/anthropic/v1/messages"): - # Already correct - pass - else: - # Check if /anthropic is already in the path - if "/anthropic" in api_base: - # /anthropic exists, ensure we end with /anthropic/v1/messages - # Extract the base URL up to and including /anthropic - parts: Final = api_base.split("/anthropic", 1) - api_base = parts[0] + "/anthropic" - else: - # /anthropic not in path, add it - api_base = api_base + "/anthropic" - # Add /v1/messages - api_base = api_base + "/v1/messages" - - return api_base + return azure_anthropic_messages_url(api_base) def _remove_scope_from_cache_control(self, anthropic_messages_request: dict) -> None: """ diff --git a/litellm/llms/azure_ai/anthropic/transformation.py b/litellm/llms/azure_ai/anthropic/transformation.py index 18a4eeb7c0a..c75213c3e2c 100644 --- a/litellm/llms/azure_ai/anthropic/transformation.py +++ b/litellm/llms/azure_ai/anthropic/transformation.py @@ -4,12 +4,23 @@ Azure Anthropic transformation config - extends AnthropicConfig with Azure authe from typing import Final +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.types.llms.openai import AllMessageValues from litellm.types.router import GenericLiteLLMParams +def azure_anthropic_messages_url(api_base: str) -> str: + base: Final = api_base.rstrip("/") + if base.endswith("/v1/messages"): + return base + path_start: Final = base.find("/", base.find("://") + len("://")) if "://" in base else 0 + anthropic_start: Final = base.find("/anthropic", path_start) if path_start >= 0 else -1 + anthropic_root: Final = base[:anthropic_start] if anthropic_start >= 0 else base + return join_api_base(anthropic_root, "anthropic/v1/messages") + + def _promote_extra_body_to_optional_params(optional_params: dict) -> None: """Promote anthropic-native passthrough keys out of ``extra_body``. diff --git a/litellm/llms/azure_ai/embed/handler.py b/litellm/llms/azure_ai/embed/handler.py index 57e6ca51ade..dc3bc6bdc5e 100644 --- a/litellm/llms/azure_ai/embed/handler.py +++ b/litellm/llms/azure_ai/embed/handler.py @@ -5,6 +5,7 @@ from urllib.parse import urlsplit, urlunsplit from openai import OpenAI import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -74,13 +75,18 @@ class AzureAIEmbedding(OpenAIChatCompletion): api_base: str | None, client: HTTPHandler | AsyncHTTPHandler | None = None, ) -> EmbeddingResponse: + if api_base is None: + raise ValueError( + "api_base is None. Please set AZURE_AI_API_BASE or dynamically via `api_base` param, " + "to make the request." + ) if client is None or not isinstance(client, AsyncHTTPHandler): client = get_async_httpx_client( llm_provider=litellm.LlmProviders.AZURE_AI, params={"timeout": timeout}, ) - url: Final = f"{api_base}/images/embeddings" + url: Final = join_api_base(api_base, "images/embeddings") response: Final = await client.post( url=url, @@ -123,7 +129,7 @@ class AzureAIEmbedding(OpenAIChatCompletion): if client is None or not isinstance(client, HTTPHandler): client = HTTPHandler(timeout=timeout, concurrent_limit=1) - url: Final = f"{api_base}/images/embeddings" + url: Final = join_api_base(api_base, "images/embeddings") response: Final = client.post( url=url, diff --git a/litellm/llms/azure_ai/image_generation/flux_transformation.py b/litellm/llms/azure_ai/image_generation/flux_transformation.py index 8b4fa78cdf4..6a7bebd9f6b 100644 --- a/litellm/llms/azure_ai/image_generation/flux_transformation.py +++ b/litellm/llms/azure_ai/image_generation/flux_transformation.py @@ -3,6 +3,7 @@ from types import MappingProxyType from typing import Final from litellm.exceptions import BadRequestError, UnsupportedParamsError +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.image_generation import GPTImageGenerationConfig from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams @@ -52,7 +53,8 @@ class AzureFoundryFluxImageGenerationConfig(GPTImageGenerationConfig): return f"{api_base}?api-version={api_version}" provider_model_path: Final = AzureFoundryFluxImageGenerationConfig.get_flux2_provider_model_path(model) - return f"{api_base}/providers/blackforestlabs/v1/{provider_model_path}?api-version={api_version}" + flux_route: Final = f"providers/blackforestlabs/v1/{provider_model_path}" + return f"{join_api_base(api_base, flux_route)}?api-version={api_version}" @staticmethod def is_flux2_model(model: str) -> bool: diff --git a/litellm/llms/azure_ai/image_generation/mai_transformation.py b/litellm/llms/azure_ai/image_generation/mai_transformation.py index 18b4b6f456a..8bb5d3e6b8d 100644 --- a/litellm/llms/azure_ai/image_generation/mai_transformation.py +++ b/litellm/llms/azure_ai/image_generation/mai_transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from litellm.exceptions import UnsupportedParamsError +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -41,9 +42,9 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): if "/mai/" in path: prefix, _, _ = path.partition("/images/") - path = f"{prefix}/images/generations" + path = join_api_base(prefix, "images/generations") else: - path = f"{path}/mai/v1/images/generations" + path = join_api_base(path, "mai/v1/images/generations") if separator: return f"{path}?{query}" @@ -63,9 +64,9 @@ class AzureFoundryMAIImageGenerationConfig(BaseImageGenerationConfig): if "/mai/" in path: prefix, _, _ = path.partition("/images/") - path = f"{prefix}/images/edits" + path = join_api_base(prefix, "images/edits") else: - path = f"{path}/mai/v1/images/edits" + path = join_api_base(path, "mai/v1/images/edits") if separator: return f"{path}?{query}" diff --git a/litellm/llms/azure_ai/vector_stores/transformation.py b/litellm/llms/azure_ai/vector_stores/transformation.py index 7638229bc32..a4777be146f 100644 --- a/litellm/llms/azure_ai/vector_stores/transformation.py +++ b/litellm/llms/azure_ai/vector_stores/transformation.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.azure.common_utils import BaseAzureLLM from litellm.llms.base_llm.vector_store.transformation import ( BaseQueryEmbeddingVectorStoreConfig, @@ -172,7 +173,8 @@ class AzureAIVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAzureLLM litellm_logging_obj.model_call_details["input"] = query_text litellm_logging_obj.model_call_details["embedding_model"] = litellm_params.get("litellm_embedding_model") litellm_logging_obj.model_call_details["top_k"] = top_k - return f"{api_base}/indexes/{vector_store_id}/docs/search?api-version=2024-07-01", { + search_route: Final = f"indexes/{vector_store_id}/docs/search" + return f"{join_api_base(api_base, search_route)}?api-version=2024-07-01", { "search": "*", "vectorQueries": [{"vector": query_vector, "fields": vector_field, "kind": "vector", "k": top_k}], "select": "id,content", diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index d74257a4a9d..a122aeb3033 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -12,6 +12,7 @@ from pydantic_core import ErrorDetails from typing_extensions import assert_never import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( @@ -582,7 +583,7 @@ class BaseDecisionsConfig(ABC): ) def get_complete_url(self, api_base: str, model: str) -> str: - return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" + return join_api_base(api_base.rstrip("/").removesuffix("/v1"), self.path) def signs_with_aws(self, api_base: str) -> bool: return False diff --git a/litellm/llms/base_llm/evals/transformation.py b/litellm/llms/base_llm/evals/transformation.py index 49124e271a6..40093953144 100644 --- a/litellm/llms/base_llm/evals/transformation.py +++ b/litellm/llms/base_llm/evals/transformation.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai_evals import ( CancelEvalResponse, @@ -79,7 +80,7 @@ class BaseEvalsAPIConfig(ABC): """ if api_base is None: raise ValueError("api_base is required") - return f"{api_base}/v1/{endpoint}" + return join_api_base(api_base, f"v1/{endpoint}") @abstractmethod def transform_create_eval_request( diff --git a/litellm/llms/base_llm/realtime/http_transformation.py b/litellm/llms/base_llm/realtime/http_transformation.py index 43a80edb493..a02054ec8cb 100644 --- a/litellm/llms/base_llm/realtime/http_transformation.py +++ b/litellm/llms/base_llm/realtime/http_transformation.py @@ -7,10 +7,11 @@ These are HTTP (not WebSocket) endpoints used by the WebRTC flow: """ from abc import ABC, abstractmethod -from typing import Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base + class BaseRealtimeHTTPConfig(ABC): """ @@ -61,7 +62,7 @@ class BaseRealtimeHTTPConfig(ABC): """Return the full URL for POST /realtime/transcription_sessions.""" base = (api_base or "").rstrip("/") base = base.removesuffix("/v1") - return f"{base}/v1/realtime/transcription_sessions" + return join_api_base(base, "v1/realtime/transcription_sessions") @abstractmethod def validate_environment( @@ -83,8 +84,7 @@ class BaseRealtimeHTTPConfig(ABC): def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: """Return the full URL for POST /realtime/calls (SDP exchange).""" - base: Final = (api_base or "").rstrip("/") - return f"{base}/v1/realtime/calls" + return join_api_base(api_base or "", "v1/realtime/calls") def get_realtime_calls_headers(self, ephemeral_key: str) -> dict: """ diff --git a/litellm/llms/base_llm/skills/transformation.py b/litellm/llms/base_llm/skills/transformation.py index ba202e3db5d..64a8e055832 100644 --- a/litellm/llms/base_llm/skills/transformation.py +++ b/litellm/llms/base_llm/skills/transformation.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Any import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.anthropic_skills import ( CreateSkillRequest, @@ -71,7 +72,7 @@ class BaseSkillsAPIConfig(ABC): """ if api_base is None: raise ValueError("api_base is required") - return f"{api_base}/v1/{endpoint}" + return join_api_base(api_base, f"v1/{endpoint}") @abstractmethod def transform_create_skill_request( diff --git a/litellm/llms/bedrock/chat/agentcore/transformation.py b/litellm/llms/bedrock/chat/agentcore/transformation.py index 48b3795e274..bb5bb246a93 100644 --- a/litellm/llms/bedrock/chat/agentcore/transformation.py +++ b/litellm/llms/bedrock/chat/agentcore/transformation.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.a2a.common_utils import extract_text_from_a2a_response from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -98,14 +99,14 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM): # Build the base endpoint URL for AgentCore # Note: We don't use get_runtime_endpoint as AgentCore has its own endpoint structure if aws_bedrock_runtime_endpoint: - base_url = aws_bedrock_runtime_endpoint + base_url = f"{aws_bedrock_runtime_endpoint}" else: base_url = f"https://bedrock-agentcore.{region}.{get_aws_dns_suffix(region)}" # Based on boto3 client.invoke_agent_runtime, the path is: # /runtimes/{URL-ENCODED-ARN}/invocations?qualifier= encoded_arn: Final = quote(agent_runtime_arn, safe="") - endpoint_url = f"{base_url}/runtimes/{encoded_arn}/invocations" + endpoint_url = join_api_base(base_url, f"runtimes/{encoded_arn}/invocations") # Add qualifier as query parameter if provided if "qualifier" in optional_params: diff --git a/litellm/llms/bedrock/chat/chat_completions/transformation.py b/litellm/llms/bedrock/chat/chat_completions/transformation.py index 907f8e50ac7..123e4a9e6b4 100644 --- a/litellm/llms/bedrock/chat/chat_completions/transformation.py +++ b/litellm/llms/bedrock/chat/chat_completions/transformation.py @@ -33,6 +33,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( inline_remote_image_urls, inline_remote_media, ) +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, bedrock_bearer_token from litellm.llms.bedrock.common_utils import ( @@ -371,12 +372,9 @@ class AmazonBedrockRuntimeChatCompletionsConfig(OpenAILikeChatConfig): ), aws_region_name=aws_region_name, ) - base: Final = proxy_endpoint_url.rstrip("/") - if base.endswith("/openai/v1/chat/completions"): - return base - if base.endswith("/openai/v1"): - return f"{base}/chat/completions" - return f"{base}/openai/v1/chat/completions" + if proxy_endpoint_url.rstrip("/").endswith("/openai/v1"): + return join_api_base(proxy_endpoint_url, "chat/completions") + return ensure_api_route(proxy_endpoint_url, "openai/v1/chat/completions") def _params_with_region_from_path( self, optional_params: dict, model: str | None diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index 7d02b9ef230..a6f9192339f 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -10,6 +10,7 @@ from litellm.anthropic_beta_headers_manager import ( ) from litellm.litellm_core_utils.get_litellm_params import stored_control_options from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm import base_model_iterator from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -363,11 +364,11 @@ class BedrockConverseLLM(BaseAWSLLM): aws_region_name=aws_region_name, ) if (stream is not None and stream is True) and not fake_stream: - endpoint_url = f"{endpoint_url}/model/{modelId}/converse-stream" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse-stream" + endpoint_url = join_api_base(endpoint_url, f"model/{modelId}/converse-stream") + proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/converse-stream") else: - endpoint_url = f"{endpoint_url}/model/{modelId}/converse" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/converse" + endpoint_url = join_api_base(endpoint_url, f"model/{modelId}/converse") + proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/converse") ## COMPLETION CALL headers = {"Content-Type": "application/json"} diff --git a/litellm/llms/bedrock/chat/invoke_agent/transformation.py b/litellm/llms/bedrock/chat/invoke_agent/transformation.py index 6877af74494..baef89912fa 100644 --- a/litellm/llms/bedrock/chat/invoke_agent/transformation.py +++ b/litellm/llms/bedrock/chat/invoke_agent/transformation.py @@ -15,7 +15,7 @@ from litellm._uuid import uuid from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError @@ -101,9 +101,10 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM): encoded_agent_alias_id: Final = encode_url_path_segment(agent_alias_id, field_name="agent_alias_id") encoded_session_id: Final = encode_url_path_segment(session_id, field_name="session_id") - endpoint_url = f"{endpoint_url}/agents/{encoded_agent_id}/agentAliases/{encoded_agent_alias_id}/sessions/{encoded_session_id}/text" - - return endpoint_url + return join_api_base( + endpoint_url, + f"agents/{encoded_agent_id}/agentAliases/{encoded_agent_alias_id}/sessions/{encoded_session_id}/text", + ) def sign_request( self, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py index 4bf1a1cba73..f1e1f3e8738 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError from litellm.llms.bedrock.request_metadata import ( @@ -97,13 +98,8 @@ class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM): # Encode model ID for ARNs (e.g., :imported-model/ -> :imported-model%2F) model_id = CommonUtils.encode_bedrock_runtime_modelid_arn(model_id) - # Build the invoke URL - if stream: - endpoint_url = f"{endpoint_url}/model/{model_id}/invoke-with-response-stream" - else: - endpoint_url = f"{endpoint_url}/model/{model_id}/invoke" - - return endpoint_url + invoke_route: Final = "invoke-with-response-stream" if stream else "invoke" + return join_api_base(endpoint_url, f"model/{model_id}/{invoke_route}") def sign_request( self, diff --git a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py index 87d4fc0b32e..51b5a99272e 100644 --- a/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py +++ b/litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( prompt_factory, render_custom_prompt_template, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call from litellm.llms.bedrock.common_utils import BedrockError @@ -136,20 +137,13 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM): aws_bedrock_runtime_endpoint: Final = optional_params.get( "aws_bedrock_runtime_endpoint", None ) # https://bedrock-runtime.{region_name}.amazonaws.com - endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( + endpoint_url, _ = self.get_runtime_endpoint( api_base=api_base, aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, aws_region_name=self._get_aws_region_name(optional_params=optional_params, model=model), ) - - if (stream is not None and stream is True) and provider != "ai21": - endpoint_url = f"{endpoint_url}/model/{modelId}/invoke-with-response-stream" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke-with-response-stream" - else: - endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" - - return endpoint_url + invoke_route: Final = "invoke-with-response-stream" if stream is True and provider != "ai21" else "invoke" + return join_api_base(endpoint_url, f"model/{modelId}/{invoke_route}") def sign_request( self, diff --git a/litellm/llms/bedrock/claude_platform/common_utils.py b/litellm/llms/bedrock/claude_platform/common_utils.py index e25e58056f5..77cae177792 100644 --- a/litellm/llms/bedrock/claude_platform/common_utils.py +++ b/litellm/llms/bedrock/claude_platform/common_utils.py @@ -5,6 +5,7 @@ import httpx import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError from litellm.secret_managers.main import get_secret_str @@ -140,9 +141,7 @@ class BedrockClaudePlatformMixin(BaseAWSLLM): if api_base is None: aws_region_name: Final = self._get_required_aws_region_name(optional_params) api_base = f"https://{CLAUDE_PLATFORM_SERVICE_NAME}.{aws_region_name}.api.aws" - if not api_base.endswith("/v1/messages"): - api_base = f"{api_base.rstrip('/')}/v1/messages" - return api_base + return ensure_api_route(api_base, "v1/messages") def sign_request( self, diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index 0cf3e895ad2..4954e646f58 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -31,6 +31,7 @@ from litellm import verbose_logger from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.litellm_core_utils.optional_imports import ensure_optional_import from litellm.litellm_core_utils.prompt_templates.common_utils import schema_property_count +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -1305,10 +1306,7 @@ def build_mantle_messages_url( """ override: Final = api_base or aws_bedrock_runtime_endpoint or _mantle_api_base_from_env() if override: - base: Final = override.rstrip("/") - if base.endswith(MANTLE_MESSAGES_PATH): - return base - return f"{base}{MANTLE_MESSAGES_PATH}" + return ensure_api_route(override, MANTLE_MESSAGES_PATH) return f"https://bedrock-mantle.{region}.api.aws{MANTLE_MESSAGES_PATH}" diff --git a/litellm/llms/bedrock/count_tokens/mantle_handler.py b/litellm/llms/bedrock/count_tokens/mantle_handler.py index 14edd39c386..34e622b5e8d 100644 --- a/litellm/llms/bedrock/count_tokens/mantle_handler.py +++ b/litellm/llms/bedrock/count_tokens/mantle_handler.py @@ -7,6 +7,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url @@ -43,7 +44,7 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM): messages_url: Final = build_mantle_messages_url( api_base=None, aws_bedrock_runtime_endpoint=None, region=aws_region_name ) - return f"{messages_url}{MANTLE_COUNT_TOKENS_SUFFIX}" + return join_api_base(messages_url, MANTLE_COUNT_TOKENS_SUFFIX) async def handle_count_tokens_request( self, diff --git a/litellm/llms/bedrock/count_tokens/transformation.py b/litellm/llms/bedrock/count_tokens/transformation.py index 7cca4f69acc..a9f95c1f9b9 100644 --- a/litellm/llms/bedrock/count_tokens/transformation.py +++ b/litellm/llms/bedrock/count_tokens/transformation.py @@ -11,6 +11,7 @@ from typing import Final, Literal from pydantic import JsonValue +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import get_bedrock_base_model @@ -236,9 +237,7 @@ class BedrockCountTokensConfig(BaseAWSLLM): aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, aws_region_name=aws_region_name, ) - endpoint: Final = f"{base_url}/model/{encoded_model_id}/count-tokens" - - return endpoint + return join_api_base(base_url, f"model/{encoded_model_id}/count-tokens") def transform_bedrock_response_to_anthropic( self, bedrock_response: Mapping[str, JsonValue] diff --git a/litellm/llms/bedrock/embed/embedding.py b/litellm/llms/bedrock/embed/embedding.py index 58cfb6cb95e..4d27d94664f 100644 --- a/litellm/llms/bedrock/embed/embedding.py +++ b/litellm/llms/bedrock/embed/embedding.py @@ -12,6 +12,7 @@ import httpx import litellm from litellm.constants import BEDROCK_EMBEDDING_PROVIDERS_LITERAL +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.cohere.embed.handler import embedding as cohere_embedding from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -512,9 +513,9 @@ class BedrockEmbedding(BaseAWSLLM): aws_region_name=aws_region_name, ) if has_async_invoke: - endpoint_url = f"{endpoint_url}/async-invoke" + endpoint_url = join_api_base(endpoint_url, "async-invoke") else: - endpoint_url = f"{endpoint_url}/model/{modelId}/invoke" + endpoint_url = join_api_base(endpoint_url, f"model/{modelId}/invoke") if batch_data is not None: if aembedding: @@ -611,7 +612,7 @@ class BedrockEmbedding(BaseAWSLLM): # Encode the ARN for use in URL path encoded_arn: Final = quote(invocation_arn, safe="") - status_url: Final = f"{endpoint_url.rstrip('/')}/async-invoke/{encoded_arn}" + status_url: Final = join_api_base(endpoint_url, f"async-invoke/{encoded_arn}") # Prepare headers for GET request headers: Final = {"Content-Type": "application/json"} diff --git a/litellm/llms/bedrock/image_edit/handler.py b/litellm/llms/bedrock/image_edit/handler.py index 53bb332f511..466993de9d1 100644 --- a/litellm/llms/bedrock/image_edit/handler.py +++ b/litellm/llms/bedrock/image_edit/handler.py @@ -14,6 +14,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.bedrock.image_edit.amazon_nova_canvas_image_edit_transformation import ( BedrockAmazonNovaCanvasImageEditConfig, ) @@ -226,7 +227,7 @@ class BedrockImageEdit(BaseAWSLLM): aws_bedrock_runtime_endpoint=boto3_credentials_info.aws_bedrock_runtime_endpoint, aws_region_name=boto3_credentials_info.aws_region_name, ) - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" + proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/invoke") data: Final = self._get_request_body( model=model, image=image, diff --git a/litellm/llms/bedrock/image_generation/image_handler.py b/litellm/llms/bedrock/image_generation/image_handler.py index e81dfe76942..abd6d40f9e9 100644 --- a/litellm/llms/bedrock/image_generation/image_handler.py +++ b/litellm/llms/bedrock/image_generation/image_handler.py @@ -8,6 +8,7 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( AmazonNovaCanvasConfig, ) @@ -248,7 +249,7 @@ class BedrockImageGeneration(BaseAWSLLM): aws_bedrock_runtime_endpoint=boto3_credentials_info.aws_bedrock_runtime_endpoint, aws_region_name=boto3_credentials_info.aws_region_name, ) - proxy_endpoint_url = f"{proxy_endpoint_url}/model/{modelId}/invoke" + proxy_endpoint_url = join_api_base(proxy_endpoint_url, f"model/{modelId}/invoke") data: Final = self._get_request_body( model=model, prompt=prompt, diff --git a/litellm/llms/bedrock/rerank/handler.py b/litellm/llms/bedrock/rerank/handler.py index 01ef5af2b67..0634275135b 100644 --- a/litellm/llms/bedrock/rerank/handler.py +++ b/litellm/llms/bedrock/rerank/handler.py @@ -6,6 +6,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -160,7 +161,7 @@ class BedrockRerankHandler(BaseAWSLLM): aws_region_name=boto3_credentials_info.aws_region_name, ) proxy_endpoint_url = proxy_endpoint_url.replace("bedrock-runtime", "bedrock-agent-runtime") - proxy_endpoint_url = f"{proxy_endpoint_url}/rerank" + proxy_endpoint_url = join_api_base(proxy_endpoint_url, "rerank") body: Final = json.dumps(data).encode("utf-8") headers = {"Content-Type": "application/json"} diff --git a/litellm/llms/bedrock/responses/transformation.py b/litellm/llms/bedrock/responses/transformation.py index 086d1211835..7cdd7b08f99 100644 --- a/litellm/llms/bedrock/responses/transformation.py +++ b/litellm/llms/bedrock/responses/transformation.py @@ -46,6 +46,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, convert_url_to_base64, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM @@ -223,9 +224,10 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): litellm_params: dict, # mutable-ok: signature fixed by the BaseResponsesAPIConfig override contract ) -> str: region: Final = self._get_aws_region_name(optional_params=litellm_params, model=None) + configured_endpoint: Final = litellm_params.get("aws_bedrock_runtime_endpoint") override: Final = ( api_base - or litellm_params.get("aws_bedrock_runtime_endpoint") + or (f"{configured_endpoint}" if configured_endpoint else None) or get_secret_str("AWS_BEDROCK_RUNTIME_ENDPOINT") ) # Partition-aware: bedrock-runtime is amazonaws.com.cn in China, and other @@ -237,7 +239,7 @@ class BedrockOpenAIResponsesConfig(BaseAWSLLM, OpenAIResponsesAPIConfig): (host[: -len(suffix)] for suffix in BEDROCK_RUNTIME_OPENAI_BASE_SUFFIXES if host.endswith(suffix)), host, ) - return f"{base}{BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH}" + return join_api_base(base, BEDROCK_RUNTIME_OPENAI_RESPONSES_PATH) def supports_native_file_search(self) -> bool: return False diff --git a/litellm/llms/bedrock/vector_stores/transformation.py b/litellm/llms/bedrock/vector_stores/transformation.py index ba8ce7e5625..96d1da19bcc 100644 --- a/litellm/llms/bedrock/vector_stores/transformation.py +++ b/litellm/llms/bedrock/vector_stores/transformation.py @@ -6,7 +6,7 @@ from urllib.parse import urlparse import httpx from litellm._logging import verbose_logger -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import BedrockError @@ -196,7 +196,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): aws_region_name=self.get_aws_region_name_for_non_llm_api_calls(aws_region_name=aws_region_name), endpoint_type="agent", ) - return f"{endpoint_url}/knowledgebases" + return join_api_base(endpoint_url, "knowledgebases") def transform_search_vector_store_request( self, @@ -212,7 +212,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM): query = " ".join(query) encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - url: Final = f"{api_base}/{encoded_vector_store_id}/retrieve" + url: Final = join_api_base(api_base, f"{encoded_vector_store_id}/retrieve") request_body: Final[dict[str, object]] = { "retrievalQuery": BedrockKBRetrievalQuery(text=query), diff --git a/litellm/llms/bedrock_mantle/messages/transformation.py b/litellm/llms/bedrock_mantle/messages/transformation.py index 78881153e91..aa6510f0491 100644 --- a/litellm/llms/bedrock_mantle/messages/transformation.py +++ b/litellm/llms/bedrock_mantle/messages/transformation.py @@ -2,6 +2,7 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Final +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM from litellm.llms.bedrock.common_utils import MANTLE_MESSAGES_PATH from litellm.llms.bedrock.messages.mantle_transformation import AmazonMantleMessagesConfig @@ -33,7 +34,7 @@ def build_mantle_native_messages_url(api_base: str | None, litellm_params: Mappi configured, ) host: Final = f"https://bedrock-mantle.{region}.api.aws" if MANTLE_HOST_RE.match(stripped) else stripped - return f"{host}{MANTLE_MESSAGES_PATH}" + return join_api_base(host, MANTLE_MESSAGES_PATH) class BedrockMantleAnthropicMessagesConfig(BedrockMantleAuthMixin, AmazonMantleMessagesConfig): diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py index 8a0ce58c8b2..3532cfbd963 100644 --- a/litellm/llms/bedrock_mantle/responses/transformation.py +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -22,6 +22,7 @@ import httpx import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items from litellm.llms.base_llm.responses.transformation import ReasoningSummaryLevel @@ -100,7 +101,7 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI if MANTLE_HOST_RE.match(base): base = f"https://bedrock-mantle.{region}.api.aws" path: Final = "/openai/v1/responses" if self.use_openai_path else "/v1/responses" - return f"{base}{path}" + return join_api_base(base, path) def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict: litellm_params = litellm_params or GenericLiteLLMParams() diff --git a/litellm/llms/black_forest_labs/image_edit/transformation.py b/litellm/llms/black_forest_labs/image_edit/transformation.py index 66eacc97fba..7cfb0daf334 100644 --- a/litellm/llms/black_forest_labs/image_edit/transformation.py +++ b/litellm/llms/black_forest_labs/image_edit/transformation.py @@ -19,7 +19,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH -from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get +from litellm.litellm_core_utils.url_utils import async_safe_get, join_api_base, safe_get from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams @@ -189,11 +189,10 @@ class BlackForestLabsImageEditConfig(BaseImageEditConfig): """ Get the complete URL for the Black Forest Labs API request. """ - base_url: str = api_base or get_secret_str("BFL_API_BASE") or DEFAULT_API_BASE - base_url = base_url.rstrip("/") + base_url: Final = api_base or get_secret_str("BFL_API_BASE") or DEFAULT_API_BASE endpoint: Final = self._get_model_endpoint(model) - return f"{base_url}{endpoint}" + return join_api_base(base_url, endpoint) def _read_image_bytes( self, diff --git a/litellm/llms/black_forest_labs/image_generation/transformation.py b/litellm/llms/black_forest_labs/image_generation/transformation.py index 9d586fba26a..928af314506 100644 --- a/litellm/llms/black_forest_labs/image_generation/transformation.py +++ b/litellm/llms/black_forest_labs/image_generation/transformation.py @@ -14,6 +14,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -202,11 +203,10 @@ class BlackForestLabsImageGenerationConfig(BaseImageGenerationConfig): """ Get the complete URL for the Black Forest Labs API request. """ - base_url: str = api_base or get_secret_str("BFL_API_BASE") or DEFAULT_API_BASE - base_url = base_url.rstrip("/") + base_url: Final = api_base or get_secret_str("BFL_API_BASE") or DEFAULT_API_BASE endpoint: Final = self._get_model_endpoint(model) - return f"{base_url}{endpoint}" + return join_api_base(base_url, endpoint) def transform_image_generation_request( self, diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index 3e7db2981ef..34953a79c60 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hid from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.responses.sse_output_recovery import ( @@ -253,8 +254,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): litellm_params: dict, ) -> str: api_base = api_base or self.authenticator.get_api_base() or CHATGPT_API_BASE - api_base = api_base.rstrip("/") - return f"{api_base}/responses" + return join_api_base(api_base, "responses") def supports_native_websocket(self) -> bool: """ChatGPT does not support native WebSocket for Responses API""" diff --git a/litellm/llms/cloudflare/chat/transformation.py b/litellm/llms/cloudflare/chat/transformation.py index ef11e51a5eb..0a76588f767 100644 --- a/litellm/llms/cloudflare/chat/transformation.py +++ b/litellm/llms/cloudflare/chat/transformation.py @@ -3,6 +3,7 @@ from typing import Final import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.secret_managers.main import ( @@ -59,7 +60,7 @@ class CloudflareChatConfig(OpenAIGPTConfig): verbose_logger.warning( "Cloudflare api_base ending in '/ai/run' is the legacy Workers AI path and no longer serves OpenAI-compatible requests; rewriting to the '/ai/v1' endpoint" ) - return f"{trimmed[: -len('/ai/run')]}/ai/v1" + return join_api_base(trimmed[: -len("/ai/run")], "ai/v1") return api_base def validate_environment( diff --git a/litellm/llms/cloudflare/common_utils.py b/litellm/llms/cloudflare/common_utils.py index 9f1293cb9c4..02855a879a2 100644 --- a/litellm/llms/cloudflare/common_utils.py +++ b/litellm/llms/cloudflare/common_utils.py @@ -3,6 +3,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import ( get_secret_str, normalize_nonempty_secret_str, @@ -34,10 +35,10 @@ def workers_ai_request_model(model: str) -> str: def workers_ai_run_url(api_base: str, model: str) -> str: normalized_api_base: Final = api_base.rstrip("/") if normalized_api_base.endswith("/ai/v1"): - return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" + return join_api_base(normalized_api_base.removesuffix("/ai/v1"), f"ai/run/{model}") if normalized_api_base.endswith("/ai/run"): return f"{normalized_api_base}/{model}" - return f"{normalized_api_base}/ai/run/{model}" + return join_api_base(normalized_api_base, f"ai/run/{model}") def unwrap_workers_ai_result(payload: object) -> object: diff --git a/litellm/llms/cohere/rerank/transformation.py b/litellm/llms/cohere/rerank/transformation.py index b6d94d4853a..8b641c24e9b 100644 --- a/litellm/llms/cohere/rerank/transformation.py +++ b/litellm/llms/cohere/rerank/transformation.py @@ -6,6 +6,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str @@ -31,11 +32,7 @@ class CohereRerankConfig(BaseRerankConfig): optional_params: dict | None = None, ) -> str: if api_base: - # Remove trailing slashes and ensure clean base URL - api_base = api_base.rstrip("/") - if not api_base.endswith("/v1/rerank"): - api_base = f"{api_base}/v1/rerank" - return api_base + return ensure_api_route(api_base, "v1/rerank") return "https://api.cohere.ai/v1/rerank" def get_supported_cohere_rerank_params(self, model: str) -> list: diff --git a/litellm/llms/cohere/rerank_v2/transformation.py b/litellm/llms/cohere/rerank_v2/transformation.py index e79357bf52f..009a220534c 100644 --- a/litellm/llms/cohere/rerank_v2/transformation.py +++ b/litellm/llms/cohere/rerank_v2/transformation.py @@ -1,5 +1,6 @@ from typing import Any, Final +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.cohere.rerank.transformation import CohereRerankConfig from litellm.types.rerank import OptionalRerankParams, RerankRequest @@ -19,11 +20,7 @@ class CohereRerankV2Config(CohereRerankConfig): optional_params: dict | None = None, ) -> str: if api_base: - # Remove trailing slashes and ensure clean base URL - api_base = api_base.rstrip("/") - if not api_base.endswith("/v2/rerank"): - api_base = f"{api_base}/v2/rerank" - return api_base + return ensure_api_route(api_base, "v2/rerank") return "https://api.cohere.ai/v2/rerank" def get_supported_cohere_rerank_params(self, model: str) -> list: diff --git a/litellm/llms/cometapi/chat/transformation.py b/litellm/llms/cometapi/chat/transformation.py index 26050700efc..2bf119a0a4a 100644 --- a/litellm/llms/cometapi/chat/transformation.py +++ b/litellm/llms/cometapi/chat/transformation.py @@ -10,6 +10,7 @@ from typing import Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -112,16 +113,14 @@ class CometAPIConfig(OpenAIGPTConfig): # Ensure we include /v1 prefix when missing if api_base.endswith("/v1"): - return f"{api_base}/{endpoint}" - if api_base.endswith("/v1/"): - return f"{api_base}{endpoint}" + return join_api_base(api_base, endpoint) # If user provided https://api.cometapi.com, add /v1 if api_base == "https://api.cometapi.com": - return f"{api_base}/v1/{endpoint}" + return join_api_base(api_base, f"v1/{endpoint}") # Generic fallback: if '/v1' not in path, add it if "/v1" not in api_base.split("//", 1)[-1]: - return f"{api_base}/v1/{endpoint}" - return f"{api_base}/{endpoint}" + return join_api_base(api_base, f"v1/{endpoint}") + return join_api_base(api_base, endpoint) def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: """ diff --git a/litellm/llms/cometapi/embed/transformation.py b/litellm/llms/cometapi/embed/transformation.py index 60f781be261..e3b5c722664 100644 --- a/litellm/llms/cometapi/embed/transformation.py +++ b/litellm/llms/cometapi/embed/transformation.py @@ -9,6 +9,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str @@ -44,9 +45,7 @@ class CometAPIEmbeddingConfig(BaseEmbeddingConfig): """ Get the complete URL for the CometAPI embedding endpoint. """ - api_base = "https://api.cometapi.com/v1" if api_base is None else api_base.rstrip("/") - complete_url: Final = f"{api_base}/embeddings" - return complete_url + return join_api_base("https://api.cometapi.com/v1" if api_base is None else api_base, "embeddings") def validate_environment( self, diff --git a/litellm/llms/cometapi/image_generation/transformation.py b/litellm/llms/cometapi/image_generation/transformation.py index 567dbfd308e..ffaabbb28cd 100644 --- a/litellm/llms/cometapi/image_generation/transformation.py +++ b/litellm/llms/cometapi/image_generation/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -77,16 +78,13 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): """ Get the complete url for the request """ - complete_url: str = ( + base_url: Final = ( api_base or get_secret_str("COMETAPI_BASE_URL") or get_secret_str("COMETAPI_API_BASE") or self.DEFAULT_BASE_URL ) - - complete_url = complete_url.rstrip("/") - complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" - return complete_url + return join_api_base(base_url, self.IMAGE_GENERATION_ENDPOINT) def validate_environment( self, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ac2ac1fe31b..a7ca0d5f49e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -59,7 +59,7 @@ from litellm.litellm_core_utils.realtime_errors import ( websocket_close_reason, ) from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -6937,7 +6937,7 @@ class BaseLLMHTTPHandler: headers={}, model=model or "", api_key=api_key ) else: - url = f"{api_base.rstrip('/')}/v1/realtime/{endpoint}" + url = join_api_base(api_base, f"v1/realtime/{endpoint}") headers = { "Authorization": f"Bearer {api_key}", "Content-Type": "application/json", @@ -7006,7 +7006,7 @@ class BaseLLMHTTPHandler: url = provider_config.get_realtime_calls_url(api_base=api_base, model=model or "", api_version=api_version) headers: dict[str, object] = provider_config.get_realtime_calls_headers(ephemeral_key=openai_ephemeral_key) else: - url = f"{api_base.rstrip('/')}/v1/realtime/calls" + url = join_api_base(api_base, "v1/realtime/calls") headers = { "Authorization": f"Bearer {openai_ephemeral_key}", } diff --git a/litellm/llms/dashscope/chat/transformation.py b/litellm/llms/dashscope/chat/transformation.py index 9a1b381a2ba..966d1a01b8d 100644 --- a/litellm/llms/dashscope/chat/transformation.py +++ b/litellm/llms/dashscope/chat/transformation.py @@ -5,6 +5,7 @@ Translates from OpenAI's `/v1/chat/completions` to DashScope's `/v1/chat/complet from collections.abc import Coroutine from typing import Final, Literal, overload +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -82,7 +83,4 @@ class DashScopeChatConfig(OpenAIGPTConfig): """ If api_base is not provided, use the default DashScope /chat/completions endpoint. """ - resolved_api_base: Final = self._resolve_chat_api_base(api_base).rstrip("/") - if resolved_api_base.endswith("/chat/completions"): - return resolved_api_base - return f"{resolved_api_base}/chat/completions" + return ensure_api_route(self._resolve_chat_api_base(api_base), "chat/completions") diff --git a/litellm/llms/dashscope/embed/transformation.py b/litellm/llms/dashscope/embed/transformation.py index e5a987ca1ad..d0c2b4dff0c 100644 --- a/litellm/llms/dashscope/embed/transformation.py +++ b/litellm/llms/dashscope/embed/transformation.py @@ -16,6 +16,7 @@ from typing import Final import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str @@ -98,10 +99,7 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - base: Final = self._resolve_embedding_api_base(api_base).rstrip("/") - if base.endswith("/embeddings"): - return base - return f"{base}/embeddings" + return ensure_api_route(self._resolve_embedding_api_base(api_base), "embeddings") def transform_embedding_request( self, diff --git a/litellm/llms/dashscope/rerank/transformation.py b/litellm/llms/dashscope/rerank/transformation.py index 4ec7d697f02..48ce4b5a07b 100644 --- a/litellm/llms/dashscope/rerank/transformation.py +++ b/litellm/llms/dashscope/rerank/transformation.py @@ -34,6 +34,7 @@ from pydantic import ConfigDict, TypeAdapter from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str @@ -94,10 +95,10 @@ class DashScopeRerankConfig(BaseRerankConfig): return cleaned if cleaned.endswith("/v1"): - return f"{cleaned}/reranks" + return join_api_base(cleaned, "reranks") # Unknown base: append /reranks rather than silently ignoring the caller's api_base. - return f"{cleaned}/reranks" + return join_api_base(cleaned, "reranks") def validate_environment( self, diff --git a/litellm/llms/dashscope/videos/transformation.py b/litellm/llms/dashscope/videos/transformation.py index 1db91cbb127..5c738423a5a 100644 --- a/litellm/llms/dashscope/videos/transformation.py +++ b/litellm/llms/dashscope/videos/transformation.py @@ -16,7 +16,7 @@ from pydantic import BaseModel, Field, StrictBool, StrictInt, StrictStr, TypeAda import litellm from litellm.exceptions import UnsupportedParamsError -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.videos.async_task_utils import ( input_reference_url, @@ -708,7 +708,7 @@ class DashScopeVideoConfig(BaseVideoConfig): if parameters: request_data["parameters"] = parameters - return request_data, (), f"{api_base}{DASHSCOPE_VIDEO_SYNTHESIS_PATH}" + return request_data, (), join_api_base(api_base, DASHSCOPE_VIDEO_SYNTHESIS_PATH) @staticmethod def _reference_input( @@ -773,7 +773,7 @@ class DashScopeVideoConfig(BaseVideoConfig): def _task_url(video_id: str, api_base: str) -> str: original_task_id: Final = extract_original_video_id(video_id) encoded_task_id: Final = encode_url_path_segment(original_task_id, field_name="video_id") - return f"{api_base}{DASHSCOPE_TASKS_PATH}/{encoded_task_id}" + return join_api_base(api_base, f"{DASHSCOPE_TASKS_PATH}/{encoded_task_id}") def transform_video_status_retrieve_response( self, diff --git a/litellm/llms/databricks/chat/transformation.py b/litellm/llms/databricks/chat/transformation.py index d46d04578cc..61b41c2a173 100644 --- a/litellm/llms/databricks/chat/transformation.py +++ b/litellm/llms/databricks/chat/transformation.py @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_name_from_message, strip_prefill_prefix_field, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import ( type_to_response_format_param, # pyright: ignore[reportUnknownVariableType] # base_utils helper returns an untyped dict @@ -339,10 +340,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig): stream: bool | None = None, ) -> str: use_ai_gateway: Final = self.uses_ai_gateway(model) - api_base = self._get_api_base(api_base, use_ai_gateway=use_ai_gateway) - url_base: Final = api_base.rstrip("/") if use_ai_gateway else api_base - complete_url: Final = f"{url_base}/chat/completions" - return complete_url + return join_api_base(self._get_api_base(api_base, use_ai_gateway=use_ai_gateway), "chat/completions") def get_supported_openai_params(self, model: str | None = None) -> list: return [ diff --git a/litellm/llms/databricks/common_utils.py b/litellm/llms/databricks/common_utils.py index 3112b2458b4..cd9432ac36f 100644 --- a/litellm/llms/databricks/common_utils.py +++ b/litellm/llms/databricks/common_utils.py @@ -20,6 +20,7 @@ from urllib.parse import urlsplit, urlunsplit from pydantic import BaseModel, ConfigDict import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -278,10 +279,10 @@ class DatabricksBase: if normalized_api_base.endswith("/ai-gateway/mlflow/v1"): return normalized_api_base if normalized_api_base.endswith("/serving-endpoints"): - return f"{normalized_api_base.removesuffix('/serving-endpoints')}/ai-gateway/mlflow/v1" + return join_api_base(normalized_api_base.removesuffix("/serving-endpoints"), "ai-gateway/mlflow/v1") api_base_parts: Final = urlsplit(normalized_api_base) if api_base_parts.path in ("", "/"): - return f"{normalized_api_base}/ai-gateway/mlflow/v1" + return join_api_base(normalized_api_base, "ai-gateway/mlflow/v1") return api_base def get_oauth_m2m_token( @@ -319,7 +320,7 @@ class DatabricksBase: api_base_parts: Final = urlsplit(api_base) workspace_url: Final = urlunsplit((api_base_parts.scheme, api_base_parts.netloc, "", "", "")) - token_url: Final = f"{workspace_url}/oidc/v1/token" + token_url: Final = join_api_base(workspace_url, "oidc/v1/token") try: response: Final = requests.post( @@ -477,5 +478,6 @@ class DatabricksBase: # Debug logging with redaction (never log actual tokens) verbose_logger.debug("Databricks request headers: %s", self.redact_headers_for_logging(headers)) - endpoint_path: Final = "" if custom_endpoint is True else DATABRICKS_ENDPOINT_PATHS[endpoint_type] - return f"{api_base}{endpoint_path}", headers + if custom_endpoint is True: + return api_base, headers + return join_api_base(api_base, DATABRICKS_ENDPOINT_PATHS[endpoint_type]), headers diff --git a/litellm/llms/databricks/decisions/transformation.py b/litellm/llms/databricks/decisions/transformation.py index c2ac1f220b8..d1698fdf5e2 100644 --- a/litellm/llms/databricks/decisions/transformation.py +++ b/litellm/llms/databricks/decisions/transformation.py @@ -6,6 +6,7 @@ from typing import Final from pydantic import TypeAdapter, ValidationError +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.types.decisions import DecisionsIRRequest, UnsupportedDecisionsRequest @@ -84,8 +85,8 @@ class DatabricksDecisionsConfig(BaseDecisionsConfig): def get_complete_url(self, api_base: str, model: str) -> str: if is_ai_decide_model(model): - return f"{databricks_workspace_host(api_base)}{_AI_DECIDE_PATH}" - return f"{api_base.rstrip('/')}/{validate_databricks_decisions_model(model)}/invocations" + return join_api_base(databricks_workspace_host(api_base), _AI_DECIDE_PATH) + return join_api_base(api_base, f"{validate_databricks_decisions_model(model)}/invocations") def transform_decisions_request( self, diff --git a/litellm/llms/databricks/responses/transformation.py b/litellm/llms/databricks/responses/transformation.py index 804c79ea79d..55c3edfefd0 100644 --- a/litellm/llms/databricks/responses/transformation.py +++ b/litellm/llms/databricks/responses/transformation.py @@ -10,6 +10,7 @@ Reference: https://docs.databricks.com/aws/en/machine-learning/foundation-model- import os from typing import TYPE_CHECKING, Any, Final +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.databricks.common_utils import DatabricksBase from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import ResponseInputParam @@ -70,8 +71,7 @@ class DatabricksResponsesAPIConfig(DatabricksBase, OpenAIResponsesAPIConfig): ) -> str: api_base = api_base or os.getenv("DATABRICKS_API_BASE") api_base = self._get_api_base(api_base) - api_base = api_base.rstrip("/") - return f"{api_base}/responses" + return join_api_base(api_base, "responses") def transform_responses_api_request( self, diff --git a/litellm/llms/deepgram/audio_transcription/transformation.py b/litellm/llms/deepgram/audio_transcription/transformation.py index a3812dac8ef..a1c3021c9f5 100644 --- a/litellm/llms/deepgram/audio_transcription/transformation.py +++ b/litellm/llms/deepgram/audio_transcription/transformation.py @@ -10,6 +10,7 @@ from httpx import Headers, Response from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -179,7 +180,6 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): ) -> str: if api_base is None: api_base = get_secret_str("DEEPGRAM_API_BASE") or "https://api.deepgram.com/v1" - api_base = api_base.rstrip("/") # Remove trailing slash if present # Build query parameters including the model all_query_params: Final = {"model": model} @@ -189,7 +189,7 @@ class DeepgramAudioTranscriptionConfig(BaseAudioTranscriptionConfig): all_query_params.update(additional_params) # Construct URL with proper query string encoding - base_url: Final = f"{api_base}/listen" + base_url: Final = join_api_base(api_base, "listen") query_string: Final = urlencode(all_query_params) url: Final = f"{base_url}?{query_string}" diff --git a/litellm/llms/deepgram/common_utils.py b/litellm/llms/deepgram/common_utils.py index 9b071ac8321..29998895bbc 100644 --- a/litellm/llms/deepgram/common_utils.py +++ b/litellm/llms/deepgram/common_utils.py @@ -8,6 +8,7 @@ import httpx import litellm from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import LlmProviders @@ -50,7 +51,7 @@ def _first_occurrences(query_string: str) -> httpx.QueryParams: def deepgram_listen_websocket_target(api_base: str | None, query_string: str) -> str: - listen_url: Final = httpx.URL(f"{(api_base or DEEPGRAM_DEFAULT_API_BASE).rstrip('/')}/listen") + listen_url: Final = httpx.URL(join_api_base(api_base or DEEPGRAM_DEFAULT_API_BASE, "listen")) websocket_url: Final = listen_url.copy_with(scheme=_WEBSOCKET_SCHEMES.get(listen_url.scheme, listen_url.scheme)) params: Final = _first_occurrences(query_string) query: Final = params if params.get("model") else params.remove("model").add("model", DEEPGRAM_LISTEN_DEFAULT_MODEL) diff --git a/litellm/llms/deepseek/chat/transformation.py b/litellm/llms/deepseek/chat/transformation.py index d3319856d29..a9a20101bb1 100644 --- a/litellm/llms/deepseek/chat/transformation.py +++ b/litellm/llms/deepseek/chat/transformation.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, extract_search_results_text, ) +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import supports_reasoning, supports_vision @@ -363,9 +364,4 @@ class DeepSeekChatConfig(OpenAIGPTConfig): """ If api_base is not provided, use the default DeepSeek /chat/completions endpoint. """ - clean_api_base: Final = (api_base or "https://api.deepseek.com/beta").rstrip("/") - - if not clean_api_base.endswith("/chat/completions"): - return f"{clean_api_base}/chat/completions" - - return clean_api_base + return ensure_api_route(api_base or "https://api.deepseek.com/beta", "chat/completions") diff --git a/litellm/llms/deepseek/messages/transformation.py b/litellm/llms/deepseek/messages/transformation.py index 6a6b57131ad..6ddee99aea3 100644 --- a/litellm/llms/deepseek/messages/transformation.py +++ b/litellm/llms/deepseek/messages/transformation.py @@ -6,6 +6,7 @@ from collections.abc import Sequence from typing import Any, Final import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) @@ -90,9 +91,9 @@ class DeepSeekAnthropicMessagesConfig(AnthropicMessagesConfig): base_url = base_url.removesuffix("/beta") if not base_url.endswith("/anthropic") and "/anthropic/" not in base_url: - base_url = f"{base_url}/anthropic" + base_url = join_api_base(base_url, "anthropic") - return f"{base_url}/v1/messages" + return join_api_base(base_url, "v1/messages") @staticmethod def _sanitize_tools_for_deepseek(tools: Any) -> object: diff --git a/litellm/llms/docker_model_runner/chat/transformation.py b/litellm/llms/docker_model_runner/chat/transformation.py index adb40311efd..68b63812248 100644 --- a/litellm/llms/docker_model_runner/chat/transformation.py +++ b/litellm/llms/docker_model_runner/chat/transformation.py @@ -10,6 +10,7 @@ from typing import Final, Literal, overload from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -104,12 +105,9 @@ class DockerModelRunnerChatConfig(OpenAIGPTConfig): if not api_base: api_base = "http://localhost:22088/engines/llama.cpp" - # Remove trailing slashes from api_base - api_base = api_base.rstrip("/") - # Build the URL: {api_base}/v1/chat/completions # api_base is expected to already contain the engine path - complete_url: Final = f"{api_base}/v1/chat/completions" + complete_url: Final = join_api_base(api_base, "v1/chat/completions") return complete_url diff --git a/litellm/llms/duckduckgo/search/transformation.py b/litellm/llms/duckduckgo/search/transformation.py index 1816a0d24e4..36893b4bd27 100644 --- a/litellm/llms/duckduckgo/search/transformation.py +++ b/litellm/llms/duckduckgo/search/transformation.py @@ -12,6 +12,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -91,7 +92,7 @@ class DuckDuckGoSearchConfig(BaseSearchConfig): if data and isinstance(data, dict) and "_duckduckgo_params" in data: params: Final = data["_duckduckgo_params"] query_string: Final = urlencode(params, doseq=True) - return f"{api_base}/?{query_string}" + return f"{join_api_base(api_base, '')}?{query_string}" return api_base diff --git a/litellm/llms/e2b/sandbox/transformation.py b/litellm/llms/e2b/sandbox/transformation.py index dbf3996248a..b5f42632eef 100644 --- a/litellm/llms/e2b/sandbox/transformation.py +++ b/litellm/llms/e2b/sandbox/transformation.py @@ -15,6 +15,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.sandbox.transformation import ( SANDBOX_MAX_OUTPUT_BYTES, BaseSandboxConfig, @@ -78,7 +79,7 @@ class E2BSandboxConfig(BaseSandboxConfig): body["metadata"] = metadata response: Final = await self._http(client).post( - url=f"{base}/sandboxes", + url=join_api_base(base, "sandboxes"), headers={"X-API-Key": key, "Content-Type": "application/json"}, json=body, ) @@ -151,10 +152,10 @@ class E2BSandboxConfig(BaseSandboxConfig): dict[str, object], getattr(handle, HIDDEN_PARAMS_ATTR) ) key: Final = api_key or hidden_params.get("api_key") or self.validate_environment() - base: Final = api_base or hidden_params.get("api_base") or E2B_API_BASE + base: Final = str(api_base or hidden_params.get("api_base") or E2B_API_BASE) try: response: Final = await self._http(client).delete( - url=f"{base}/sandboxes/{handle.id}", + url=join_api_base(base, f"sandboxes/{handle.id}"), headers={"X-API-Key": key}, ) except httpx.HTTPStatusError as e: diff --git a/litellm/llms/edenai/chat/transformation.py b/litellm/llms/edenai/chat/transformation.py index ae5da5d1cfd..480b382076d 100644 --- a/litellm/llms/edenai/chat/transformation.py +++ b/litellm/llms/edenai/chat/transformation.py @@ -23,7 +23,7 @@ from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, ModelResponseStream, Usage -from ..common_utils import EdenAIException, reported_cost, resolve_api_base, resolve_api_key +from ..common_utils import EdenAIException, endpoint_url, reported_cost, resolve_api_base, resolve_api_key if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -138,7 +138,7 @@ class EdenAIChatConfig(OpenAIGPTConfig): def get_models( self, api_key: str | None = None, api_base: str | None = None ) -> list[str]: # mutable-ok: inherited contract - response: Final = litellm.module_level_client.get(url=f"{self.get_api_base(api_base)}/models") + response: Final = litellm.module_level_client.get(url=endpoint_url(api_base, "models")) if not response.is_success: raise EdenAIException(status_code=response.status_code, message=response.text, headers=response.headers) catalog: Final = _EdenAIModelCatalog.model_validate(response.json()) diff --git a/litellm/llms/edenai/common_utils.py b/litellm/llms/edenai/common_utils.py index 3216f798fbe..60c83f7ea75 100644 --- a/litellm/llms/edenai/common_utils.py +++ b/litellm/llms/edenai/common_utils.py @@ -11,6 +11,7 @@ from pydantic import AliasChoices, Field, ValidationError import litellm from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.base import LiteLLMBaseModel @@ -74,7 +75,7 @@ def json_headers( def endpoint_url(api_base: str | None, path: str) -> str: - return f"{resolve_api_base(api_base).rstrip('/')}/{path}" + return join_api_base(resolve_api_base(api_base), path) def pick(params: Mapping[str, object], keys: Container[str]) -> Mapping[str, object]: diff --git a/litellm/llms/elevenlabs/audio_transcription/transformation.py b/litellm/llms/elevenlabs/audio_transcription/transformation.py index 04d2717d64f..5a673b5ac41 100644 --- a/litellm/llms/elevenlabs/audio_transcription/transformation.py +++ b/litellm/llms/elevenlabs/audio_transcription/transformation.py @@ -10,6 +10,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -165,12 +166,7 @@ class ElevenLabsAudioTranscriptionConfig(BaseAudioTranscriptionConfig): ) -> str: if api_base is None: api_base = get_secret_str("ELEVENLABS_API_BASE") or "https://api.elevenlabs.io" - api_base = api_base.rstrip("/") # Remove trailing slash if present - - # ElevenLabs speech-to-text endpoint - url: Final = f"{api_base}/v1/speech-to-text" - - return url + return join_api_base(api_base, "v1/speech-to-text") def validate_environment( self, diff --git a/litellm/llms/elevenlabs/text_to_speech/transformation.py b/litellm/llms/elevenlabs/text_to_speech/transformation.py index eb93543df46..f910ef8c057 100644 --- a/litellm/llms/elevenlabs/text_to_speech/transformation.py +++ b/litellm/llms/elevenlabs/text_to_speech/transformation.py @@ -11,7 +11,7 @@ import httpx from httpx import Headers import litellm -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, @@ -297,15 +297,14 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig): """ Construct the ElevenLabs endpoint URL, including path voice_id and query params. """ - base_url = api_base or get_secret_str("ELEVENLABS_API_BASE") or self.TTS_BASE_URL - base_url = base_url.rstrip("/") + base_url: Final = api_base or get_secret_str("ELEVENLABS_API_BASE") or self.TTS_BASE_URL voice_id: Final = litellm_params.get(self.ELEVENLABS_VOICE_ID_KEY) if not isinstance(voice_id, str) or not voice_id.strip(): raise ValueError("ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`.") encoded_voice_id: Final = encode_url_path_segment(voice_id, field_name="voice_id") - url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{encoded_voice_id}" + url = join_api_base(base_url, f"{self.TTS_ENDPOINT_PATH}/{encoded_voice_id}") query_params: Final = litellm_params.get(self.ELEVENLABS_QUERY_PARAMS_KEY, {}) if query_params: diff --git a/litellm/llms/exa_ai/search/transformation.py b/litellm/llms/exa_ai/search/transformation.py index 89073409599..73e341d0033 100644 --- a/litellm/llms/exa_ai/search/transformation.py +++ b/litellm/llms/exa_ai/search/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -93,11 +94,7 @@ class ExaAISearchConfig(BaseSearchConfig): """ Get complete URL for Search endpoint. """ - resolved_api_base: Final = api_base or get_secret_str("EXA_API_BASE") or self.EXA_AI_API_BASE - clean_api_base: Final = resolved_api_base.rstrip("/") - if clean_api_base.endswith("/search"): - return clean_api_base - return f"{clean_api_base}/search" + return ensure_api_route(api_base or get_secret_str("EXA_API_BASE") or self.EXA_AI_API_BASE, "search") def transform_search_request( self, diff --git a/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py b/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py index 6d58caeb384..344d9d3c75b 100644 --- a/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py +++ b/litellm/llms/fal_ai/image_edit/flux_lora_depth_transformation.py @@ -4,6 +4,7 @@ from typing import Final from httpx._types import RequestFiles +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.router import GenericLiteLLMParams @@ -45,8 +46,8 @@ class FalAIFluxLoraDepthEditConfig(FalAIImageEditConfig): api_base: str | None, litellm_params: dict, # mutable-ok: base class contract ) -> str: - base_url: Final = (api_base or get_secret_str("FAL_AI_API_BASE") or DEFAULT_BASE_URL).rstrip("/") - return f"{base_url}/{FLUX_LORA_DEPTH_ENDPOINT}" + base_url: Final = api_base or get_secret_str("FAL_AI_API_BASE") or DEFAULT_BASE_URL + return join_api_base(base_url, FLUX_LORA_DEPTH_ENDPOINT) def transform_image_edit_request( self, diff --git a/litellm/llms/fal_ai/image_generation/transformation.py b/litellm/llms/fal_ai/image_generation/transformation.py index 0bb5dada071..79913d5c651 100644 --- a/litellm/llms/fal_ai/image_generation/transformation.py +++ b/litellm/llms/fal_ai/image_generation/transformation.py @@ -5,6 +5,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -85,12 +86,10 @@ class FalAIBaseConfig(BaseImageGenerationConfig): Some providers need `model` in `api_base` """ - complete_url: str = api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL - - complete_url = complete_url.rstrip("/") - if self.IMAGE_GENERATION_ENDPOINT: - complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" - return complete_url + base_url: Final = api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL + if not self.IMAGE_GENERATION_ENDPOINT: + return base_url.rstrip("/") + return join_api_base(base_url, self.IMAGE_GENERATION_ENDPOINT) def validate_environment( self, diff --git a/litellm/llms/fal_ai/videos/transformation.py b/litellm/llms/fal_ai/videos/transformation.py index 3136763e1be..86aa6388ff0 100644 --- a/litellm/llms/fal_ai/videos/transformation.py +++ b/litellm/llms/fal_ai/videos/transformation.py @@ -10,7 +10,7 @@ import httpx from httpx._types import FileContent, RequestFiles from pydantic import TypeAdapter -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_httpx.http_handler import ( @@ -445,7 +445,7 @@ class FalAIVideoConfig(BaseVideoConfig): request_id, model_id = self._decode_video_id(video_id) encoded_request_id: Final[str] = encode_url_path_segment(request_id, field_name="video_id") return ( - f"{api_base.rstrip('/')}/{_queue_request_base_path(model_id)}/requests/{encoded_request_id}/status", + join_api_base(api_base, f"{_queue_request_base_path(model_id)}/requests/{encoded_request_id}/status"), {}, ) @@ -541,7 +541,7 @@ class FalAIVideoConfig(BaseVideoConfig): request_id, model_id = self._decode_video_id(video_id) encoded_request_id: Final[str] = encode_url_path_segment(request_id, field_name="video_id") return ( - f"{api_base.rstrip('/')}/{_queue_request_base_path(model_id)}/requests/{encoded_request_id}", + join_api_base(api_base, f"{_queue_request_base_path(model_id)}/requests/{encoded_request_id}"), {}, ) diff --git a/litellm/llms/fastcrw/search/transformation.py b/litellm/llms/fastcrw/search/transformation.py index 8f6f8d87300..27db6a95170 100644 --- a/litellm/llms/fastcrw/search/transformation.py +++ b/litellm/llms/fastcrw/search/transformation.py @@ -15,6 +15,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -83,11 +84,7 @@ class FastCRWSearchConfig(BaseSearchConfig): """ Get complete URL for Search endpoint. """ - resolved_api_base: Final = api_base or get_secret_str("CRW_API_BASE") or self.FASTCRW_API_BASE - clean_api_base: Final = resolved_api_base.rstrip("/") - if clean_api_base.endswith("/search"): - return clean_api_base - return f"{clean_api_base}/search" + return ensure_api_route(api_base or get_secret_str("CRW_API_BASE") or self.FASTCRW_API_BASE, "search") def transform_search_request( self, diff --git a/litellm/llms/firecrawl/search/transformation.py b/litellm/llms/firecrawl/search/transformation.py index ee99ab24413..6d1e2b22918 100644 --- a/litellm/llms/firecrawl/search/transformation.py +++ b/litellm/llms/firecrawl/search/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -85,11 +86,7 @@ class FirecrawlSearchConfig(BaseSearchConfig): """ Get complete URL for Search endpoint. """ - resolved_api_base: Final = api_base or get_secret_str("FIRECRAWL_API_BASE") or self.FIRECRAWL_API_BASE - clean_api_base: Final = resolved_api_base.rstrip("/") - if clean_api_base.endswith("/search"): - return clean_api_base - return f"{clean_api_base}/search" + return ensure_api_route(api_base or get_secret_str("FIRECRAWL_API_BASE") or self.FIRECRAWL_API_BASE, "search") def transform_search_request( self, diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index 14ae59d4f85..fa5794d71b3 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -17,6 +17,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_response_only_fields, unpack_legacy_defs, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, @@ -826,7 +827,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): base = api_base.rstrip("/") base = base.removesuffix("/v1") response: Final = litellm.module_level_client.get( - url=f"{base}/v1/accounts/{account_id}/models", + url=join_api_base(base, f"v1/accounts/{account_id}/models"), headers={"Authorization": f"Bearer {api_key}"}, ) diff --git a/litellm/llms/fireworks_ai/rerank/transformation.py b/litellm/llms/fireworks_ai/rerank/transformation.py index 4eff64e351f..964f85c821e 100644 --- a/litellm/llms/fireworks_ai/rerank/transformation.py +++ b/litellm/llms/fireworks_ai/rerank/transformation.py @@ -12,6 +12,7 @@ from pydantic import ConfigDict, TypeAdapter from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.fireworks_ai.common_utils import FireworksAIMixin from litellm.types.llms.base import LiteLLMBaseModel @@ -61,9 +62,9 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig): api_base = api_base.rstrip("/") if not api_base.endswith("/rerank"): if api_base.endswith("/v1") or api_base.endswith("/inference/v1"): - api_base = f"{api_base}/rerank" + api_base = join_api_base(api_base, "rerank") else: - api_base = f"{api_base}/inference/v1/rerank" + api_base = join_api_base(api_base, "inference/v1/rerank") return api_base return "https://api.fireworks.ai/inference/v1/rerank" diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index 30eaa5a7ca2..db234c09930 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -6,6 +6,7 @@ from urllib.parse import unquote import httpx from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.fireworks_ai.common_utils import ( FIREWORKS_FORWARD_USER_ID_PARAM, get_fireworks_forwarded_user_id, @@ -150,8 +151,8 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): return dict(pinned) def get_complete_url(self, api_base: str | None, litellm_params: Mapping[str, object]) -> str: - base: Final = (api_base or get_secret_str("FIREWORKS_API_BASE") or FIREWORKS_AI_DEFAULT_API_BASE).rstrip("/") - return f"{base}/responses" + base: Final = api_base or get_secret_str("FIREWORKS_API_BASE") or FIREWORKS_AI_DEFAULT_API_BASE + return join_api_base(base, "responses") def transform_responses_api_request( self, diff --git a/litellm/llms/gdc/chat/transformation.py b/litellm/llms/gdc/chat/transformation.py index 6eac3ac79cd..cd9ffe9e03e 100644 --- a/litellm/llms/gdc/chat/transformation.py +++ b/litellm/llms/gdc/chat/transformation.py @@ -13,6 +13,7 @@ from urllib.parse import urlsplit from typing_extensions import ReadOnly, TypedDict, Unpack import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.types.llms.openai import AllMessageValues @@ -140,7 +141,7 @@ class GDCGeminiConfig(OpenAILikeChatConfig): project = self._validate_path_id(project, "vertex_project", model) location = self._validate_path_id(location, "vertex_location", model) - return f"{api_base}/v1/projects/{project}/locations/{location}/chat/completions" + return join_api_base(api_base, f"v1/projects/{project}/locations/{location}/chat/completions") def _read_env_bool(self, val: bool | str | None, env_var: str, default: bool = True) -> bool | str: def _parse(s: str) -> bool | str: diff --git a/litellm/llms/gemini/agents/transformation.py b/litellm/llms/gemini/agents/transformation.py index cfdb55f2048..c19bd9ce87c 100644 --- a/litellm/llms/gemini/agents/transformation.py +++ b/litellm/llms/gemini/agents/transformation.py @@ -15,6 +15,7 @@ from typing import Any, Final import httpx from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.agents.transformation import BaseAgentsAPIConfig from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo from litellm.types.agents import ( @@ -67,7 +68,7 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): return "v1beta" def _base_url(self, api_base: str | None) -> str: - return f"{GeminiModelInfo.get_api_base(api_base)}/{self.api_version}" + return join_api_base(GeminiModelInfo.get_api_base(api_base), self.api_version) # ------------------------------------------------------------------ # # Shared helpers # @@ -90,7 +91,7 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): api_base: str | None, litellm_params: Mapping[str, object], ) -> str: - return f"{self._base_url(api_base)}/agents" + return join_api_base(self._base_url(api_base), "agents") def validate_environment( self, @@ -177,7 +178,7 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): api_base: str | None, litellm_params: Mapping[str, object], ) -> tuple[str, dict[str, object]]: - url: Final = f"{self._base_url(api_base)}/agents" + url: Final = join_api_base(self._base_url(api_base), "agents") params: Final[dict[str, object]] = {} if litellm_params.get("page_size"): params["pageSize"] = litellm_params["page_size"] @@ -210,7 +211,7 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): api_base: str | None, litellm_params: Mapping[str, object], ) -> tuple[str, dict[str, object]]: - url: Final = f"{self._base_url(api_base)}/agents/{name}" + url: Final = join_api_base(self._base_url(api_base), f"agents/{name}") return url, {} def transform_get_response( @@ -239,7 +240,7 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): api_base: str | None, litellm_params: Mapping[str, object], ) -> str: - return f"{self._base_url(api_base)}/agents/{name}" + return join_api_base(self._base_url(api_base), f"agents/{name}") def transform_delete_response( self, @@ -265,7 +266,7 @@ class GeminiAgentsConfig(BaseAgentsAPIConfig): api_base: str | None, litellm_params: Mapping[str, object], ) -> tuple[str, dict[str, object]]: - url: Final = f"{self._base_url(api_base)}/agents/{name}/versions" + url: Final = join_api_base(self._base_url(api_base), f"agents/{name}/versions") params: Final[dict[str, object]] = {} if litellm_params.get("page_size"): params["pageSize"] = litellm_params["page_size"] diff --git a/litellm/llms/gemini/audio_transcription/transformation.py b/litellm/llms/gemini/audio_transcription/transformation.py index 48a345a4940..ea116a7e9de 100644 --- a/litellm/llms/gemini/audio_transcription/transformation.py +++ b/litellm/llms/gemini/audio_transcription/transformation.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.audio_utils.utils import ( normalize_transcription_language_to_bcp47, process_audio_file, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -104,8 +105,7 @@ class GeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig): litellm_params: Mapping[str, object], stream: bool | None = None, ) -> str: - resolved_api_base: Final = GeminiModelInfo.get_api_base(api_base) - return f"{resolved_api_base}/v1beta/interactions" + return join_api_base(GeminiModelInfo.get_api_base(api_base), "v1beta/interactions") def transform_audio_transcription_request( self, diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 8ee7a463828..13df3921d13 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -11,6 +11,7 @@ from pydantic import JsonValue, TypeAdapter import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -377,7 +378,7 @@ class GeminiModelInfo(BaseLLMModelInfo): return "v1beta" @staticmethod - def get_api_base(api_base: str | None = None) -> str | None: + def get_api_base(api_base: str | None = None) -> str: return api_base or get_secret_str("GEMINI_API_BASE") or "https://generativelanguage.googleapis.com" @staticmethod @@ -400,13 +401,13 @@ class GeminiModelInfo(BaseLLMModelInfo): api_base = GeminiModelInfo.get_api_base(api_base) api_key = GeminiModelInfo.get_api_key(api_key) endpoint: Final = f"/{self.api_version}/models" - if api_base is None or api_key is None: + if api_key is None: raise ValueError( "GEMINI_API_BASE or GEMINI_API_KEY/GOOGLE_API_KEY is not set. Please set the environment variable, to query Gemini's `/models` endpoint." ) response: Final = litellm.module_level_client.get( - url=f"{api_base}{endpoint}", + url=join_api_base(api_base, endpoint), headers={"x-goog-api-key": api_key}, ) diff --git a/litellm/llms/gemini/count_tokens/handler.py b/litellm/llms/gemini/count_tokens/handler.py index 1c9d09330a8..4ed47323d4f 100644 --- a/litellm/llms/gemini/count_tokens/handler.py +++ b/litellm/llms/gemini/count_tokens/handler.py @@ -6,6 +6,7 @@ from pydantic import ConfigDict, RootModel, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.types.utils import LlmProviders @@ -65,7 +66,7 @@ class GoogleAIStudioTokenCounter: Construct the URL for the Google Gen AI Studio countTokens endpoint. """ base_url: Final = api_base or "https://generativelanguage.googleapis.com" - return f"{base_url}/v1beta/models/{model}:countTokens" + return join_api_base(base_url, f"v1beta/models/{model}:countTokens") def __init__(self, client: AsyncHTTPHandler | None = None) -> None: super().__init__() diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index 2c62e04c5a3..f9a7e230ea9 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -15,7 +15,7 @@ from typing_extensions import ReadOnly, Required from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, LiteLLMLoggingObj, @@ -106,8 +106,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): if not final_api_key: raise ValueError("api_key is required") - url: Final = f"{api_base}/{endpoint}" - return url + return join_api_base(api_base, endpoint) def get_supported_openai_params(self, model: str) -> list[OpenAICreateFileRequestOptionalParams]: return [] @@ -235,10 +234,11 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): file_part: Final = self._normalize_gemini_file_id(file_id) - api_base = self.get_api_base(litellm_params.get("api_base")) or "https://generativelanguage.googleapis.com" - api_base = api_base.rstrip("/") + api_base: Final = ( + self.get_api_base(litellm_params.get("api_base")) or "https://generativelanguage.googleapis.com" + ) - url: Final = f"{api_base}/v1beta/{file_part}" + url: Final = join_api_base(api_base, f"v1beta/{file_part}") # API key is passed via x-goog-api-key header (set in validate_environment) return url, {} @@ -345,7 +345,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): file_name: Final = self._normalize_gemini_file_id(file_id) # Construct the delete URL - url: Final = f"{api_base}/v1beta/{file_name}" + url: Final = join_api_base(api_base, f"v1beta/{file_name}") # Add API key as header (Google AI Studio uses x-goog-api-key header) params: Final[dict] = {} diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index 31d3963c70c..450c38a2206 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -7,6 +7,7 @@ import httpx from httpx._types import RequestFiles from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.gemini.common_utils import ( get_gemini_image_generation_config, @@ -79,9 +80,8 @@ class GeminiImageEditConfig(BaseImageEditConfig): api_base: str | None, litellm_params: dict, ) -> str: - base_url = api_base or get_secret_str("GEMINI_API_BASE") or self.DEFAULT_BASE_URL - base_url = base_url.rstrip("/") - return f"{base_url}/models/{model}:generateContent" + base_url: Final = api_base or get_secret_str("GEMINI_API_BASE") or self.DEFAULT_BASE_URL + return join_api_base(base_url, f"models/{model}:generateContent") def transform_image_edit_request( self, diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index fe82a9cae81..1aa3d074b1a 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -88,14 +89,12 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): """ complete_url: str = api_base or get_secret_str("GEMINI_API_BASE") or self.DEFAULT_BASE_URL - complete_url = complete_url.rstrip("/") - # Gemini Flash Image Preview models use generateContent endpoint if is_gemini_image_model(model): - complete_url = f"{complete_url}/models/{model}:generateContent" + complete_url = join_api_base(complete_url, f"models/{model}:generateContent") else: # All other Imagen models use predict endpoint - complete_url = f"{complete_url}/models/{model}:predict" + complete_url = join_api_base(complete_url, f"models/{model}:predict") return complete_url diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index a58037f691f..6f3e9a56b92 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -16,7 +16,7 @@ from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import process_response_headers -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo from litellm.types.interactions import ( @@ -153,10 +153,8 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): if not api_key: raise ValueError("Google API key is required. Set GOOGLE_API_KEY or GEMINI_API_KEY environment variable.") - if stream: - return f"{api_base}/{self.api_version}/interactions?alt=sse" - - return f"{api_base}/{self.api_version}/interactions" + collection_url: Final = join_api_base(api_base, f"{self.api_version}/interactions") + return f"{collection_url}?alt=sse" if stream else collection_url def transform_request( self, @@ -304,7 +302,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): raise ValueError("Google API key is required") encoded_interaction_id: Final = encode_url_path_segment(interaction_id, field_name="interaction_id") return ( - f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}", + join_api_base(resolved_api_base, f"{self.api_version}/interactions/{encoded_interaction_id}"), {}, ) @@ -344,7 +342,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): raise ValueError("Google API key is required") encoded_interaction_id: Final = encode_url_path_segment(interaction_id, field_name="interaction_id") return ( - f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}", + join_api_base(resolved_api_base, f"{self.api_version}/interactions/{encoded_interaction_id}"), {}, ) @@ -375,7 +373,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): raise ValueError("Google API key is required") encoded_interaction_id: Final = encode_url_path_segment(interaction_id, field_name="interaction_id") return ( - f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}/cancel", + join_api_base(resolved_api_base, f"{self.api_version}/interactions/{encoded_interaction_id}/cancel"), {}, ) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index f4e2941670a..9af7edeac86 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -13,6 +13,7 @@ import litellm from litellm import verbose_logger from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, @@ -212,7 +213,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # so the API key must remain as a query parameter here. This is an accepted # limitation; httpx is not used for WebSocket so MaskedHTTPStatusError # already covers the main leak vector. - return f"{api_base}/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={api_key}" + live_url: Final = join_api_base( + api_base, "ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent" + ) + return f"{live_url}?key={api_key}" def map_model_turn_event(self, model_turn: HttpxContentType) -> OpenAIRealtimeEventTypes: """ diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index 34bd9660c43..11e619eb4c7 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from typing_extensions import ReadOnly, TypedDict +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.gemini.common_utils import ( GeminiError, @@ -141,15 +142,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): Note: This returns the base URL WITHOUT the API key. The API key will be appended to specific endpoint URLs in the transform methods. """ - if api_base is None: - api_base = GeminiModelInfo.get_api_base() - - if api_base is None: - raise ValueError("GEMINI_API_BASE is not set") - - # Ensure we're using the v1beta version for File Search - api_version: Final = "v1beta" - return f"{api_base}/{api_version}" + return join_api_base(GeminiModelInfo.get_api_base(api_base), "v1beta") def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> GeminiError: """Return Gemini-specific error class.""" @@ -187,7 +180,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): api_key: Final = litellm_params.get("api_key") or GeminiModelInfo.get_api_key() if not api_key: raise ValueError("GEMINI_API_KEY or GOOGLE_API_KEY is required") - url: Final = f"{api_base}/models/{model}:generateContent" + url: Final = join_api_base(api_base, f"models/{model}:generateContent") # Build file_search tool configuration (using snake_case as per Gemini docs) file_search_config: Final[dict[str, object]] = {"file_search_store_names": [vector_store_id]} @@ -328,7 +321,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): """ Transform create request to Gemini's fileSearchStores format. """ - url: Final = f"{api_base}/fileSearchStores" + url: Final = join_api_base(api_base, "fileSearchStores") # API key is passed via x-goog-api-key header (set in validate_environment) diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 53ca5ed4adc..0656095927c 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -8,6 +8,7 @@ from httpx._types import RequestFiles import litellm from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.vertex_ai.videos.transformation import veo_video_count_from_parameters from litellm.secret_managers.main import get_secret_str @@ -241,7 +242,7 @@ class GeminiVideoConfig(BaseVideoConfig): return api_base.rstrip("/") model_name: Final = model.replace("gemini/", "") - url: Final = f"{api_base.rstrip('/')}/v1beta/models/{model_name}:predictLongRunning" + url: Final = join_api_base(api_base, f"v1beta/models/{model_name}:predictLongRunning") return url @@ -377,7 +378,7 @@ class GeminiVideoConfig(BaseVideoConfig): GET https://generativelanguage.googleapis.com/v1beta/{operation_name} """ operation_name: Final = extract_original_video_id(video_id) - url: Final = f"{api_base.rstrip('/')}/v1beta/{operation_name}" + url: Final = join_api_base(api_base, f"v1beta/{operation_name}") params: Final[dict[str, object]] = {} return url, params @@ -451,7 +452,7 @@ class GeminiVideoConfig(BaseVideoConfig): """ operation_name: Final = extract_original_video_id(video_id) - status_url: Final = f"{api_base.rstrip('/')}/v1beta/{operation_name}" + status_url: Final = join_api_base(api_base, f"v1beta/{operation_name}") client: Final = litellm.module_level_client status_response: Final = client.get(url=status_url, headers=headers) status_response.raise_for_status() diff --git a/litellm/llms/gigachat/chat/transformation.py b/litellm/llms/gigachat/chat/transformation.py index 1fe7de0ba85..936e6339865 100644 --- a/litellm/llms/gigachat/chat/transformation.py +++ b/litellm/llms/gigachat/chat/transformation.py @@ -17,6 +17,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.gigachat.utils import convert_usage, get_api_base from litellm.secret_managers.main import get_secret_str @@ -108,8 +109,7 @@ class GigaChatConfig(BaseConfig): stream: bool | None = None, ) -> str: """Get complete API URL for chat completions.""" - base: Final = get_api_base(api_base) - return f"{base}/chat/completions" + return join_api_base(get_api_base(api_base), "chat/completions") def validate_environment( self, diff --git a/litellm/llms/gigachat/embedding/transformation.py b/litellm/llms/gigachat/embedding/transformation.py index e0e746427ad..30ca7147865 100644 --- a/litellm/llms/gigachat/embedding/transformation.py +++ b/litellm/llms/gigachat/embedding/transformation.py @@ -16,6 +16,7 @@ from pydantic import ConfigDict, TypeAdapter from litellm import LlmProviders from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.gigachat.utils import get_api_base @@ -98,8 +99,7 @@ class GigaChatEmbeddingConfig(BaseEmbeddingConfig): stream: bool | None = None, ) -> str: """Get the complete URL for embeddings endpoint.""" - base: Final = get_api_base(api_base) - return f"{base}/embeddings" + return join_api_base(get_api_base(api_base), "embeddings") def transform_embedding_request( self, diff --git a/litellm/llms/gigachat/file_handler.py b/litellm/llms/gigachat/file_handler.py index c051d693720..aff0465d1ec 100644 --- a/litellm/llms/gigachat/file_handler.py +++ b/litellm/llms/gigachat/file_handler.py @@ -13,6 +13,7 @@ from collections.abc import Mapping from typing import Final from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, get_httpx_client, @@ -125,8 +126,7 @@ def upload_file_sync( access_token: Final = get_access_token(credentials=credentials, litellm_params=litellm_params) # Upload to GigaChat - base_url: Final = get_api_base(api_base) - upload_url: Final = f"{base_url}/files" + upload_url: Final = join_api_base(get_api_base(api_base), "files") client: Final = get_httpx_client(params={"ssl_verify": False}) response: Final = client.post( @@ -191,8 +191,7 @@ async def upload_file_async( access_token: Final = await get_access_token_async(credentials=credentials, litellm_params=litellm_params) # Upload to GigaChat - base_url: Final = get_api_base(api_base) - upload_url: Final = f"{base_url}/files" + upload_url: Final = join_api_base(get_api_base(api_base), "files") client: Final = get_async_httpx_client( llm_provider=LlmProviders.GIGACHAT, diff --git a/litellm/llms/gigachat/passthrough/transformation.py b/litellm/llms/gigachat/passthrough/transformation.py index f16d263fd4d..872eee6dbf9 100644 --- a/litellm/llms/gigachat/passthrough/transformation.py +++ b/litellm/llms/gigachat/passthrough/transformation.py @@ -40,7 +40,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): if base_target_url is None: raise Exception("GigaChat api base not found") - complete_url: Final = f"{base_target_url}/{endpoint.lstrip('/')}" + complete_url: Final = f"{base_target_url.rstrip('/')}/{endpoint.lstrip('/')}" return ( httpx.URL(complete_url), @@ -193,8 +193,7 @@ class GigaChatPassthroughConfig(BasePassthroughConfig): @staticmethod def get_api_base(api_base: str | None = None) -> str | None: - resolved_api_base: Final = api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL - return resolved_api_base.rstrip("/") if resolved_api_base else None + return api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL @staticmethod def get_api_key( diff --git a/litellm/llms/gigachat/utils.py b/litellm/llms/gigachat/utils.py index ab1890ad73b..55ca4260221 100644 --- a/litellm/llms/gigachat/utils.py +++ b/litellm/llms/gigachat/utils.py @@ -21,5 +21,5 @@ def convert_usage(usage_data: Mapping[str, int]) -> Usage: ) -def get_api_base(api_base: str | None = None) -> str | None: +def get_api_base(api_base: str | None = None) -> str: return (api_base or get_secret_str("GIGACHAT_API_BASE") or GIGACHAT_BASE_URL).rstrip("/") diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index 7f9209afd65..f16564ba505 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -20,6 +20,7 @@ from pydantic import ConfigDict, TypeAdapter from litellm._logging import verbose_logger from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.types.llms.openai import AllEmbeddingInputValues @@ -121,7 +122,7 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig): user_session: Final = require_github_copilot_user_session( cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict ) - effective_api_base = ( + effective_api_base: Final = ( (user_session.api_base if user_session is not None else None) or api_base or self.authenticator.get_api_base() @@ -129,11 +130,7 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig): or DEFAULT_GITHUB_COPILOT_API_BASE ) - # Remove trailing slashes - effective_api_base = effective_api_base.rstrip("/") - - # Return the embeddings endpoint - return f"{effective_api_base}/embeddings" + return join_api_base(effective_api_base, "embeddings") def transform_embedding_request( self, diff --git a/litellm/llms/github_copilot/messages/transformation.py b/litellm/llms/github_copilot/messages/transformation.py index be138755f85..56c3693a02c 100644 --- a/litellm/llms/github_copilot/messages/transformation.py +++ b/litellm/llms/github_copilot/messages/transformation.py @@ -1,6 +1,7 @@ from typing import Any, Final, cast # noqa: TID251 # narrows untyped request dicts at the session boundary from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) @@ -140,5 +141,5 @@ class GithubCopilotAnthropicMessagesConfig(AnthropicMessagesConfig): or api_base or self.authenticator.get_api_base() or DEFAULT_GITHUB_COPILOT_API_BASE - ).rstrip("/") - return resolved if resolved.endswith("/v1/messages") else f"{resolved}/v1/messages" + ) + return ensure_api_route(resolved, "v1/messages") diff --git a/litellm/llms/github_copilot/responses/transformation.py b/litellm/llms/github_copilot/responses/transformation.py index d468889b626..bf0b8c79068 100644 --- a/litellm/llms/github_copilot/responses/transformation.py +++ b/litellm/llms/github_copilot/responses/transformation.py @@ -20,6 +20,7 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import ( ResponseInputParam, @@ -266,7 +267,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): user_session: Final = require_github_copilot_user_session( cast("dict[str, object]", litellm_params) # cast-ok: litellm_params arrives as an untyped request dict ) - effective_api_base = ( + effective_api_base: Final = ( (user_session.api_base if user_session is not None else None) or api_base or self.authenticator.get_api_base() @@ -274,11 +275,7 @@ class GithubCopilotResponsesAPIConfig(OpenAIResponsesAPIConfig): or DEFAULT_GITHUB_COPILOT_API_BASE ) - # Remove trailing slashes - effective_api_base = effective_api_base.rstrip("/") - - # Return the responses endpoint - return f"{effective_api_base}/responses" + return join_api_base(effective_api_base, "responses") def _handle_reasoning_item(self, item: dict[str, object]) -> dict[str, object]: """ diff --git a/litellm/llms/gradient_ai/chat/transformation.py b/litellm/llms/gradient_ai/chat/transformation.py index fcee55a27e9..0ba5ca45c2a 100644 --- a/litellm/llms/gradient_ai/chat/transformation.py +++ b/litellm/llms/gradient_ai/chat/transformation.py @@ -1,5 +1,6 @@ from typing import Final, Literal +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, @@ -105,10 +106,10 @@ class GradientAIConfig(OpenAILikeChatConfig): ) -> str: clean_api_base: Final = (api_base or "").rstrip("/") if clean_api_base and clean_api_base != GRADIENT_AI_SERVERLESS_ENDPOINT: - return f"{clean_api_base}/api/v1/chat/completions" + return join_api_base(clean_api_base, "api/v1/chat/completions") agent_endpoint: Final = (get_secret_str("GRADIENT_AI_AGENT_ENDPOINT") or "").rstrip("/") if agent_endpoint and agent_endpoint != GRADIENT_AI_SERVERLESS_ENDPOINT: - return f"{agent_endpoint}/api/v1/chat/completions" + return join_api_base(agent_endpoint, "api/v1/chat/completions") return f"{GRADIENT_AI_SERVERLESS_ENDPOINT}/v1/chat/completions" def _get_openai_compatible_provider_info( diff --git a/litellm/llms/heroku/chat/transformation.py b/litellm/llms/heroku/chat/transformation.py index acd630c85d0..002078dd04b 100644 --- a/litellm/llms/heroku/chat/transformation.py +++ b/litellm/llms/heroku/chat/transformation.py @@ -6,11 +6,12 @@ this is OpenAI compatible - no translation needed / occurs import os from collections.abc import Coroutine -from typing import Final, Literal, overload +from typing import Literal, overload from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, ) +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.types.llms.openai import AllMessageValues @@ -78,9 +79,4 @@ class HerokuChatConfig(OpenAIGPTConfig): "No api base was set. Please provide an api_base, or set the HEROKU_API_BASE environment variable." ) - clean_api_base: Final = resolved_base.rstrip("/") - - if not clean_api_base.endswith("/v1/chat/completions"): - return f"{clean_api_base}/v1/chat/completions" - - return clean_api_base + return ensure_api_route(resolved_base, "v1/chat/completions") diff --git a/litellm/llms/hosted_vllm/embedding/transformation.py b/litellm/llms/hosted_vllm/embedding/transformation.py index 3b54145a67b..d72a8d06f21 100644 --- a/litellm/llms/hosted_vllm/embedding/transformation.py +++ b/litellm/llms/hosted_vllm/embedding/transformation.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str @@ -81,14 +82,7 @@ class HostedVLLMEmbeddingConfig(BaseEmbeddingConfig): if api_base is None: raise ValueError("api_base is required for hosted_vllm embeddings") - # Remove trailing slashes - api_base = api_base.rstrip("/") - - # Ensure the URL ends with /embeddings - if not api_base.endswith("/embeddings"): - api_base = f"{api_base}/embeddings" - - return api_base + return ensure_api_route(api_base, "embeddings") def transform_embedding_request( self, diff --git a/litellm/llms/hosted_vllm/image_edit/transformation.py b/litellm/llms/hosted_vllm/image_edit/transformation.py index 6804d3a0fb8..8e05b6dfb46 100644 --- a/litellm/llms/hosted_vllm/image_edit/transformation.py +++ b/litellm/llms/hosted_vllm/image_edit/transformation.py @@ -1,5 +1,6 @@ from typing import Final +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig from litellm.secret_managers.main import get_secret_str @@ -39,5 +40,5 @@ class HostedVLLMImageEditConfig(OpenAIImageEditConfig): ) trimmed: Final = resolved_api_base.rstrip("/") if trimmed.endswith("/v1"): - return f"{trimmed}/images/edits" - return f"{trimmed}/v1/images/edits" + return join_api_base(trimmed, "images/edits") + return join_api_base(trimmed, "v1/images/edits") diff --git a/litellm/llms/hosted_vllm/rerank/transformation.py b/litellm/llms/hosted_vllm/rerank/transformation.py index a463fc1d3c9..4a5729945b4 100644 --- a/litellm/llms/hosted_vllm/rerank/transformation.py +++ b/litellm/llms/hosted_vllm/rerank/transformation.py @@ -12,6 +12,7 @@ from pydantic import ValidationError from litellm._uuid import uuid from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str @@ -62,7 +63,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig): if api_base.endswith("/v1/rerank"): api_base = api_base.replace("/v1/rerank", "/rerank") elif not api_base.endswith("/rerank"): - api_base = f"{api_base}/rerank" + api_base = join_api_base(api_base, "rerank") return api_base raise ValueError("api_base must be provided for Hosted VLLM rerank") diff --git a/litellm/llms/hosted_vllm/responses/transformation.py b/litellm/llms/hosted_vllm/responses/transformation.py index 20d755a2273..8489bc0da84 100644 --- a/litellm/llms/hosted_vllm/responses/transformation.py +++ b/litellm/llms/hosted_vllm/responses/transformation.py @@ -8,6 +8,7 @@ the chat completions → responses conversion pipeline. from typing import Final +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams @@ -64,9 +65,9 @@ class HostedVLLMResponsesAPIConfig(OpenAIResponsesAPIConfig): # If api_base already ends with /v1, append /responses # Otherwise append /v1/responses if api_base.endswith("/v1"): - return f"{api_base}/responses" + return join_api_base(api_base, "responses") - return f"{api_base}/v1/responses" + return join_api_base(api_base, "v1/responses") def supports_native_websocket(self) -> bool: """Hosted vLLM does not support native WebSocket for Responses API""" diff --git a/litellm/llms/hosted_vllm/transcriptions/transformation.py b/litellm/llms/hosted_vllm/transcriptions/transformation.py index 9a8c67fadc0..785ccd52a69 100644 --- a/litellm/llms/hosted_vllm/transcriptions/transformation.py +++ b/litellm/llms/hosted_vllm/transcriptions/transformation.py @@ -6,6 +6,7 @@ from typing import Final import httpx +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, ) @@ -40,11 +41,7 @@ class HostedVLLMAudioTranscriptionConfig(OpenAIWhisperAudioTranscriptionConfig): stream: bool | None = None, ) -> str: if api_base: - # Remove trailing slashes and ensure clean base URL - api_base = api_base.rstrip("/") - if not api_base.endswith("/v1/audio/transcriptions"): - api_base = f"{api_base}/v1/audio/transcriptions" - return api_base + return ensure_api_route(api_base, "v1/audio/transcriptions") raise ValueError("api_base must be provided for Hosted VLLM rerank") def transform_audio_transcription_request( diff --git a/litellm/llms/hosted_vllm/videos/transformation.py b/litellm/llms/hosted_vllm/videos/transformation.py index 22e4f4876fb..30ae6835263 100644 --- a/litellm/llms/hosted_vllm/videos/transformation.py +++ b/litellm/llms/hosted_vllm/videos/transformation.py @@ -10,7 +10,7 @@ from urllib.parse import urlparse from httpx._types import FileTypes, RequestFiles from litellm.images.utils import ImageEditRequestUtils -from litellm.litellm_core_utils.url_utils import SSRFError, validate_url +from litellm.litellm_core_utils.url_utils import SSRFError, join_api_base, validate_url from litellm.llms.openai.videos.transformation import OpenAIVideoConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams @@ -177,8 +177,8 @@ class HostedVLLMVideoConfig(OpenAIVideoConfig): ) trimmed: Final = resolved_api_base.rstrip("/") if trimmed.endswith("/v1"): - return f"{trimmed}/videos" - return f"{trimmed}/v1/videos" + return join_api_base(trimmed, "videos") + return join_api_base(trimmed, "v1/videos") def transform_video_create_request( self, diff --git a/litellm/llms/huggingface/chat/transformation.py b/litellm/llms/huggingface/chat/transformation.py index b45c706f8bf..3fcb738eacb 100644 --- a/litellm/llms/huggingface/chat/transformation.py +++ b/litellm/llms/huggingface/chat/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest if TYPE_CHECKING: @@ -29,11 +30,11 @@ def _build_chat_completion_url(model_url: str) -> str: # Append /chat/completions if not already present if model_url.endswith("/v1"): - model_url += "/chat/completions" + model_url = join_api_base(model_url, "chat/completions") # Append /v1/chat/completions if not already present if not model_url.endswith("/chat/completions"): - model_url += "/v1/chat/completions" + model_url = join_api_base(model_url, "v1/chat/completions") return model_url diff --git a/litellm/llms/huggingface/rerank/transformation.py b/litellm/llms/huggingface/rerank/transformation.py index 334c60ee848..1217a2540d4 100644 --- a/litellm/llms/huggingface/rerank/transformation.py +++ b/litellm/llms/huggingface/rerank/transformation.py @@ -7,6 +7,7 @@ from typing_extensions import TypedDict import litellm from litellm._uuid import uuid +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str @@ -69,15 +70,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig): """ Get the complete URL for the API call, including the /rerank suffix if necessary. """ - # Get base URL from api_base or default - base_url = self.get_api_base(model=model, api_base=api_base) - - # Remove trailing slashes and ensure we have the /rerank endpoint - base_url = base_url.rstrip("/") - if not base_url.endswith("/rerank"): - base_url = f"{base_url}/rerank" - - return base_url + return ensure_api_route(self.get_api_base(model=model, api_base=api_base), "rerank") def get_supported_cohere_rerank_params(self, model: str) -> list: return [ diff --git a/litellm/llms/infinity/embedding/transformation.py b/litellm/llms/infinity/embedding/transformation.py index f289918cb3c..a467c56fc74 100644 --- a/litellm/llms/infinity/embedding/transformation.py +++ b/litellm/llms/infinity/embedding/transformation.py @@ -5,6 +5,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str @@ -36,11 +37,7 @@ class InfinityEmbeddingConfig(BaseEmbeddingConfig): ) -> str: if api_base is None: raise ValueError("api_base is required for Infinity embeddings") - # Remove trailing slashes and ensure clean base URL - api_base = api_base.rstrip("/") - if not api_base.endswith("/embeddings"): - api_base = f"{api_base}/embeddings" - return api_base + return ensure_api_route(api_base, "embeddings") def validate_environment( self, diff --git a/litellm/llms/infinity/rerank/transformation.py b/litellm/llms/infinity/rerank/transformation.py index e089b3fecbe..07f3bb2c26b 100644 --- a/litellm/llms/infinity/rerank/transformation.py +++ b/litellm/llms/infinity/rerank/transformation.py @@ -13,6 +13,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.cohere.rerank.transformation import CohereRerankConfig from litellm.secret_managers.main import get_secret_str from litellm.types.rerank import ( @@ -61,11 +62,7 @@ class InfinityRerankConfig(CohereRerankConfig): ) -> str: if api_base is None: raise ValueError("api_base is required for Infinity rerank") - # Remove trailing slashes and ensure clean base URL - api_base = api_base.rstrip("/") - if not api_base.endswith("/rerank"): - api_base = f"{api_base}/rerank" - return api_base + return ensure_api_route(api_base, "rerank") def validate_environment( self, diff --git a/litellm/llms/jina_ai/embedding/transformation.py b/litellm/llms/jina_ai/embedding/transformation.py index 9b156e5ae21..04ff8f941a3 100644 --- a/litellm/llms/jina_ai/embedding/transformation.py +++ b/litellm/llms/jina_ai/embedding/transformation.py @@ -15,6 +15,7 @@ from pydantic import ConfigDict, TypeAdapter from litellm import LlmProviders from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm import BaseEmbeddingConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str @@ -115,12 +116,7 @@ class JinaAIEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - if api_base: - clean_api_base: Final = api_base.rstrip("/") - if not clean_api_base.endswith("/embeddings"): - return f"{clean_api_base}/embeddings" - return clean_api_base - return "https://api.jina.ai/v1/embeddings" + return ensure_api_route(api_base or "https://api.jina.ai/v1", "embeddings") def transform_embedding_request( self, diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py index 1781082d36f..0938cc0d07b 100644 --- a/litellm/llms/langflow/chat/transformation.py +++ b/litellm/llms/langflow/chat/transformation.py @@ -9,6 +9,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -107,9 +108,8 @@ class LangFlowConfig(BaseConfig): "api_base is required for LangFlow. Set it via LANGFLOW_API_BASE env var or api_base parameter." ) - api_base = api_base.rstrip("/") flow_id: Final = quote(self._get_flow_id(model, optional_params), safe="") - return f"{api_base}/api/v1/run/{flow_id}" + return join_api_base(api_base, f"api/v1/run/{flow_id}") def _get_last_user_message(self, messages: list[AllMessageValues]) -> str: """Extract the text of the last user message to use as input_value.""" diff --git a/litellm/llms/langgraph/chat/transformation.py b/litellm/llms/langgraph/chat/transformation.py index 4f93f435957..ce624eecc0e 100644 --- a/litellm/llms/langgraph/chat/transformation.py +++ b/litellm/llms/langgraph/chat/transformation.py @@ -18,6 +18,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.langgraph.chat.sse_iterator import LangGraphSSEStreamIterator from litellm.types.llms.openai import AllMessageValues @@ -116,14 +117,11 @@ class LangGraphConfig(BaseConfig): "api_base is required for LangGraph. Set it via LANGGRAPH_API_BASE env var or api_base parameter." ) - # Remove trailing slash if present - api_base = api_base.rstrip("/") - # Choose endpoint based on streaming mode if stream: - return f"{api_base}/runs/stream" + return join_api_base(api_base, "runs/stream") else: - return f"{api_base}/runs/wait" + return join_api_base(api_base, "runs/wait") def _get_assistant_id(self, model: str, optional_params: dict) -> str: """ diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index a740050dd94..01fdc0b8bf8 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -12,6 +12,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, @@ -105,7 +106,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): # Getting the list of models from lemonade try: response: Final = litellm.module_level_client.get( - url=f"{api_base}/models", + url=join_api_base(api_base, "models"), headers=self._get_auth_headers(api_key), ) except Exception as e: @@ -193,7 +194,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): try: response: Final = litellm.module_level_client.get( - url=f"{api_base}/models/{encoded_model}", + url=join_api_base(str(api_base), f"models/{encoded_model}"), headers=self._get_auth_headers(api_key), ) response.raise_for_status() diff --git a/litellm/llms/linkup/search/transformation.py b/litellm/llms/linkup/search/transformation.py index e219a227b1f..fd1ca792302 100644 --- a/litellm/llms/linkup/search/transformation.py +++ b/litellm/llms/linkup/search/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -87,11 +88,7 @@ class LinkupSearchConfig(BaseSearchConfig): """ Get complete URL for Search endpoint. """ - resolved_api_base: Final = api_base or get_secret_str("LINKUP_API_BASE") or self.LINKUP_API_BASE - clean_api_base: Final = resolved_api_base.rstrip("/") - if clean_api_base.endswith("/search"): - return clean_api_base - return f"{clean_api_base}/search" + return ensure_api_route(api_base or get_secret_str("LINKUP_API_BASE") or self.LINKUP_API_BASE, "search") def transform_search_request( self, diff --git a/litellm/llms/litellm_proxy/image_edit/transformation.py b/litellm/llms/litellm_proxy/image_edit/transformation.py index 9e5a2a2c343..f319e31ff3f 100644 --- a/litellm/llms/litellm_proxy/image_edit/transformation.py +++ b/litellm/llms/litellm_proxy/image_edit/transformation.py @@ -1,3 +1,4 @@ +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.image_edit.transformation import OpenAIImageEditConfig from litellm.secret_managers.main import get_secret_str @@ -21,5 +22,4 @@ class LiteLLMProxyImageEditConfig(OpenAIImageEditConfig): api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE") if api_base is None: raise ValueError("api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`") - api_base = api_base.rstrip("/") - return f"{api_base}/images/edits" + return join_api_base(api_base, "images/edits") diff --git a/litellm/llms/litellm_proxy/image_generation/transformation.py b/litellm/llms/litellm_proxy/image_generation/transformation.py index fcddeabfc85..05e536eece5 100644 --- a/litellm/llms/litellm_proxy/image_generation/transformation.py +++ b/litellm/llms/litellm_proxy/image_generation/transformation.py @@ -1,3 +1,4 @@ +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.image_generation.gpt_transformation import ( GPTImageGenerationConfig, ) @@ -33,5 +34,4 @@ class LiteLLMProxyImageGenerationConfig(GPTImageGenerationConfig): api_base = api_base or get_secret_str("LITELLM_PROXY_API_BASE") if api_base is None: raise ValueError("api_base not set for LiteLLM Proxy route. Set in env via `LITELLM_PROXY_API_BASE`") - api_base = api_base.rstrip("/") - return f"{api_base}/images/generations" + return join_api_base(api_base, "images/generations") diff --git a/litellm/llms/litellm_proxy/responses/transformation.py b/litellm/llms/litellm_proxy/responses/transformation.py index 2928b07665a..3092b49b743 100644 --- a/litellm/llms/litellm_proxy/responses/transformation.py +++ b/litellm/llms/litellm_proxy/responses/transformation.py @@ -5,6 +5,7 @@ LiteLLM Proxy supports the OpenAI Responses API natively when the underlying mod This config enables pass-through behavior to the proxy's /v1/responses endpoint. """ +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.utils import LlmProviders @@ -40,10 +41,7 @@ class LiteLLMProxyResponsesAPIConfig(OpenAIResponsesAPIConfig): "Set via api_base parameter or LITELLM_PROXY_API_BASE environment variable" ) - # Remove trailing slashes - api_base = api_base.rstrip("/") - - return f"{api_base}/responses" + return join_api_base(api_base, "responses") def supports_native_websocket(self) -> bool: """LiteLLM Proxy does not support native WebSocket for Responses API""" diff --git a/litellm/llms/manus/files/transformation.py b/litellm/llms/manus/files/transformation.py index 94b0e62199d..991aa4cbc32 100644 --- a/litellm/llms/manus/files/transformation.py +++ b/litellm/llms/manus/files/transformation.py @@ -21,7 +21,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import ( BaseFilesConfig, @@ -140,8 +140,8 @@ class ManusFilesConfig(BaseFilesConfig): # Manus API uses /v1/files endpoint if api_base.endswith("/v1"): - return f"{api_base}/files" - return f"{api_base}/v1/files" + return join_api_base(api_base, "files") + return join_api_base(api_base, "v1/files") def get_error_class( self, @@ -423,7 +423,7 @@ class ManusFilesConfig(BaseFilesConfig): litellm_params=litellm_params, ) encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id") - return f"{api_base}/{encoded_file_id}/content", {} + return join_api_base(api_base, f"{encoded_file_id}/content"), {} def transform_file_content_response( self, diff --git a/litellm/llms/manus/responses/transformation.py b/litellm/llms/manus/responses/transformation.py index 9785a50bbce..ed06391a260 100644 --- a/litellm/llms/manus/responses/transformation.py +++ b/litellm/llms/manus/responses/transformation.py @@ -10,7 +10,7 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, ) -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.openai.common_utils import OpenAIError from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str @@ -119,8 +119,8 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig): # Manus API uses /v1/responses endpoint (OpenAI-compatible) if api_base.endswith("/v1"): - return f"{api_base}/responses" - return f"{api_base}/v1/responses" + return join_api_base(api_base, "responses") + return join_api_base(api_base, "v1/responses") def transform_responses_api_request( self, diff --git a/litellm/llms/milvus/vector_stores/transformation.py b/litellm/llms/milvus/vector_stores/transformation.py index 70d3649debd..67de3a06af8 100644 --- a/litellm/llms/milvus/vector_stores/transformation.py +++ b/litellm/llms/milvus/vector_stores/transformation.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.vector_store.transformation import ( BaseQueryEmbeddingVectorStoreConfig, VectorStoreEmbeddingExecutor, @@ -185,7 +186,7 @@ class MilvusVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig): } litellm_logging_obj.model_call_details["input"] = query_text litellm_logging_obj.model_call_details["embedding_model"] = litellm_params.get("litellm_embedding_model") - return f"{api_base}/v2/vectordb/entities/search", { + return join_api_base(api_base, "v2/vectordb/entities/search"), { "collectionName": vector_store_id, "data": [query_vector], "annsField": "book_intro_vector", diff --git a/litellm/llms/minimax/chat/transformation.py b/litellm/llms/minimax/chat/transformation.py index c5aa8811f02..f397bf7fbb9 100644 --- a/litellm/llms/minimax/chat/transformation.py +++ b/litellm/llms/minimax/chat/transformation.py @@ -5,6 +5,7 @@ MiniMax OpenAI transformation config - extends OpenAI chat config for MiniMax's from typing import Final import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam @@ -55,17 +56,14 @@ class MinimaxChatConfig(OpenAIGPTConfig): Override to ensure we use MiniMax's endpoint. """ # Get the base URL (either provided or default MiniMax endpoint) - base_url: Final = self.get_api_base(api_base=api_base) + base_url: Final = self.get_api_base(api_base=api_base).rstrip("/") # Ensure it ends with /chat/completions if base_url.endswith("/chat/completions"): return base_url - elif base_url.endswith("/v1"): - return f"{base_url}/chat/completions" - elif base_url.endswith("/"): - return f"{base_url}v1/chat/completions" - else: - return f"{base_url}/v1/chat/completions" + if base_url.endswith("/v1"): + return join_api_base(base_url, "chat/completions") + return join_api_base(base_url, "v1/chat/completions") def remove_cache_control_flag_from_messages_and_tools( self, diff --git a/litellm/llms/minimax/messages/transformation.py b/litellm/llms/minimax/messages/transformation.py index d62b88a24c6..f9f3502237c 100644 --- a/litellm/llms/minimax/messages/transformation.py +++ b/litellm/llms/minimax/messages/transformation.py @@ -2,9 +2,10 @@ MiniMax Anthropic transformation config - extends AnthropicConfig for MiniMax's Anthropic-compatible API """ -from typing import Any, Final # noqa: TID251 # override below must mirror the legacy base signature +from typing import Any # noqa: TID251 # override below must mirror the legacy base signature import litellm +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) @@ -82,15 +83,4 @@ class MinimaxMessagesConfig(AnthropicMessagesConfig): Get the complete URL for MiniMax API. Override to ensure we use MiniMax's endpoint, not Anthropic's. """ - # Get the base URL (either provided or default MiniMax endpoint) - base_url: Final = self.get_api_base(api_base=api_base) - - # If the base URL already includes the full path, return it - if base_url.endswith("/v1/messages"): - return base_url - - # Otherwise append the messages endpoint - if base_url.endswith("/"): - return f"{base_url}v1/messages" - else: - return f"{base_url}/v1/messages" + return ensure_api_route(self.get_api_base(api_base=api_base), "v1/messages") diff --git a/litellm/llms/minimax/text_to_speech/transformation.py b/litellm/llms/minimax/text_to_speech/transformation.py index 1cabc43eb1c..6f2cb36ed03 100644 --- a/litellm/llms/minimax/text_to_speech/transformation.py +++ b/litellm/llms/minimax/text_to_speech/transformation.py @@ -13,6 +13,7 @@ from httpx import Headers from pydantic import ConfigDict, TypeAdapter import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, @@ -404,10 +405,5 @@ class MinimaxTextToSpeechConfig(BaseTextToSpeechConfig): """ Construct the MiniMax endpoint URL. """ - base_url = api_base or get_secret_str("MINIMAX_API_BASE") or self.TTS_BASE_URL - base_url = base_url.rstrip("/") - - # MiniMax uses a simple endpoint path - url: Final = f"{base_url}{self.TTS_ENDPOINT_PATH}" - - return url + base_url: Final = api_base or get_secret_str("MINIMAX_API_BASE") or self.TTS_BASE_URL + return join_api_base(base_url, self.TTS_ENDPOINT_PATH) diff --git a/litellm/llms/mistral/audio_speech/transformation.py b/litellm/llms/mistral/audio_speech/transformation.py index 5a8960d0898..ac0e1a33620 100644 --- a/litellm/llms/mistral/audio_speech/transformation.py +++ b/litellm/llms/mistral/audio_speech/transformation.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, @@ -118,9 +119,7 @@ class MistralTextToSpeechConfig(BaseTextToSpeechConfig): api_base: str | None, litellm_params: Mapping[str, object], ) -> str: - configured_base: Final = (api_base or self.TTS_BASE_URL).rstrip("/") - versioned_base: Final = configured_base if configured_base.endswith("/v1") else f"{configured_base}/v1" - return f"{versioned_base}/audio/speech" + return join_api_base(ensure_api_route(api_base or self.TTS_BASE_URL, "v1"), "audio/speech") def transform_text_to_speech_request( self, diff --git a/litellm/llms/mistral/audio_transcription/transformation.py b/litellm/llms/mistral/audio_transcription/transformation.py index 4fce7a95350..3eb9e122265 100644 --- a/litellm/llms/mistral/audio_transcription/transformation.py +++ b/litellm/llms/mistral/audio_transcription/transformation.py @@ -9,6 +9,7 @@ from typing import Final import httpx from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -57,8 +58,7 @@ class MistralAudioTranscriptionConfig(BaseAudioTranscriptionConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - api_base = "https://api.mistral.ai/v1" if api_base is None else api_base.rstrip("/") - return f"{api_base}/audio/transcriptions" + return join_api_base("https://api.mistral.ai/v1" if api_base is None else api_base, "audio/transcriptions") def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return MistralAudioTranscriptionException( diff --git a/litellm/llms/mistral/batches/transformation.py b/litellm/llms/mistral/batches/transformation.py index 86de2836048..18f5df9b6e9 100644 --- a/litellm/llms/mistral/batches/transformation.py +++ b/litellm/llms/mistral/batches/transformation.py @@ -18,7 +18,7 @@ from openai.types.batch_error import BatchError from pydantic import ConfigDict from typing_extensions import NotRequired, ReadOnly, TypedDict -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.base import LiteLLMBaseModel @@ -156,7 +156,7 @@ class MistralBatchesConfig(BaseBatchesConfig): litellm_params: Mapping[str, object], data: CreateBatchRequest, ) -> str: - return f"{get_mistral_api_base(api_base)}/v1/batch/jobs" + return join_api_base(get_mistral_api_base(api_base), "v1/batch/jobs") def transform_create_batch_request( self, @@ -199,7 +199,10 @@ class MistralBatchesConfig(BaseBatchesConfig): api_key: Final = litellm_params.get("api_key") request: Final = MistralPresignedRequest( method="GET", - url=f"{get_mistral_api_base(api_base if isinstance(api_base, str) else None)}/v1/batch/jobs/{encoded_batch_id}", + url=join_api_base( + get_mistral_api_base(api_base if isinstance(api_base, str) else None), + f"v1/batch/jobs/{encoded_batch_id}", + ), headers=get_mistral_auth_headers(_NO_HEADERS, api_key if isinstance(api_key, str) else None), ) return dict(request) diff --git a/litellm/llms/mistral/chat/transformation.py b/litellm/llms/mistral/chat/transformation.py index b8a6cb1beb9..2c0acacff23 100644 --- a/litellm/llms/mistral/chat/transformation.py +++ b/litellm/llms/mistral/chat/transformation.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( strip_litellm_internal_message_fields, strip_none_values_from_message, ) +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.openai.chat.gpt_transformation import ( OpenAIChatCompletionStreamingHandler, OpenAIGPTConfig, @@ -220,8 +221,7 @@ class MistralConfig(OpenAIGPTConfig): ) # if api_base does not end with /v1 we add it - if api_base is not None and not api_base.endswith("/v1"): # Mistral always needs a /v1 at the end - api_base = api_base + "/v1" + api_base = ensure_api_route(api_base, "v1") dynamic_api_key: Final = ( api_key or get_secret_str("MISTRAL_AZURE_API_KEY") # for Azure AI Mistral diff --git a/litellm/llms/mistral/files/transformation.py b/litellm/llms/mistral/files/transformation.py index 86bce763917..908567e88fb 100644 --- a/litellm/llms/mistral/files/transformation.py +++ b/litellm/llms/mistral/files/transformation.py @@ -18,7 +18,7 @@ from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj from litellm.types.llms.base import LiteLLMBaseModel @@ -130,11 +130,11 @@ class MistralFilesConfig(BaseFilesConfig): litellm_params: Mapping[str, object], stream: bool | None = None, ) -> str: - return f"{get_mistral_api_base(api_base)}/v1/files" + return join_api_base(get_mistral_api_base(api_base), "v1/files") def _file_url(self, file_id: str, litellm_params: Mapping[str, object], suffix: str = "") -> str: encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id") - return f"{_api_base_from(litellm_params)}/v1/files/{encoded_file_id}{suffix}" + return join_api_base(_api_base_from(litellm_params), f"v1/files/{encoded_file_id}{suffix}") def get_error_class( self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers @@ -233,7 +233,7 @@ class MistralFilesConfig(BaseFilesConfig): optional_params: Mapping[str, object], litellm_params: Mapping[str, object], ) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature - url: Final = f"{_api_base_from(litellm_params)}/v1/files" + url: Final = join_api_base(_api_base_from(litellm_params), "v1/files") if not purpose: return url, _NO_QUERY_PARAMS return url, {"purpose": _to_mistral_purpose(purpose)} diff --git a/litellm/llms/modelscope/chat/transformation.py b/litellm/llms/modelscope/chat/transformation.py index c54437b9a21..ec48b344bd9 100644 --- a/litellm/llms/modelscope/chat/transformation.py +++ b/litellm/llms/modelscope/chat/transformation.py @@ -7,6 +7,7 @@ from typing import Final, Literal, cast, overload from typing_extensions import override +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -86,9 +87,4 @@ class ModelScopeChatConfig(OpenAIGPTConfig): """ If api_base is not provided, use the default ModelScope /chat/completions endpoint. """ - clean_api_base: Final = (api_base or self.DEFAULT_BASE_URL).rstrip("/") - - if not clean_api_base.endswith("/chat/completions"): - return f"{clean_api_base}/chat/completions" - - return clean_api_base + return ensure_api_route(api_base or self.DEFAULT_BASE_URL, "chat/completions") diff --git a/litellm/llms/modelscope/image_generation/transformation.py b/litellm/llms/modelscope/image_generation/transformation.py index 08cdbacdad0..36c8705f256 100644 --- a/litellm/llms/modelscope/image_generation/transformation.py +++ b/litellm/llms/modelscope/image_generation/transformation.py @@ -13,6 +13,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from typing_extensions import override +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, @@ -90,11 +91,10 @@ class ModelScopeImageGenerationConfig(BaseImageGenerationConfig): """ Get the complete URL for the ModelScope image generation API request. """ - base_url: str = api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL - base_url = base_url.rstrip("/") + base_url: Final = api_base or get_secret_str("MODELSCOPE_API_BASE") or self.DEFAULT_BASE_URL # Return the images endpoint - return f"{base_url}/images/generations" + return join_api_base(base_url, "images/generations") @override def validate_environment( diff --git a/litellm/llms/mongodb/vector_stores/transformation.py b/litellm/llms/mongodb/vector_stores/transformation.py index 0d931ee78cb..7bf9d8b8441 100644 --- a/litellm/llms/mongodb/vector_stores/transformation.py +++ b/litellm/llms/mongodb/vector_stores/transformation.py @@ -9,6 +9,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError from litellm.exceptions import AuthenticationError, BadRequestError, ServiceUnavailableError, Timeout +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.vector_store.transformation import ( BaseQueryEmbeddingVectorStoreConfig, @@ -283,7 +284,7 @@ class MongoDBVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig): raise config_error("The embedding model must return a non-empty, finite query vector.") limit: Final = cls._limit(optional_params) return ( - f"{api_base}/v1/vector_stores/{quote(vector_store_id, safe='')}/search", + join_api_base(api_base, f"v1/vector_stores/{quote(vector_store_id, safe='')}/search"), { "query": query_text, "query_vector": tuple(vector), diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index 93496e775db..698db8b7c25 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -9,6 +9,7 @@ import litellm from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, ) +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.utils import supports_reasoning @@ -93,12 +94,7 @@ class MoonshotChatConfig(OpenAIGPTConfig): """ If api_base is not provided, use the default Moonshot AI /chat/completions endpoint. """ - clean_api_base: Final = (api_base or "https://api.moonshot.ai/v1").rstrip("/") - - if not clean_api_base.endswith("/chat/completions"): - return f"{clean_api_base}/chat/completions" - - return clean_api_base + return ensure_api_route(api_base or "https://api.moonshot.ai/v1", "chat/completions") def get_supported_openai_params(self, model: str) -> list: """ diff --git a/litellm/llms/nimble/search/transformation.py b/litellm/llms/nimble/search/transformation.py index 30cfc38eb87..b4d9be909c3 100644 --- a/litellm/llms/nimble/search/transformation.py +++ b/litellm/llms/nimble/search/transformation.py @@ -13,6 +13,7 @@ from typing import TYPE_CHECKING, Final import httpx from pydantic import ConfigDict, TypeAdapter, ValidationError +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, @@ -124,10 +125,7 @@ class NimbleSearchConfig(BaseSearchConfig): data: dict[str, object] | list[dict[str, object]] | None = None, # mutable-ok: base signature **kwargs: object, # kwargs-ok: BaseSearchConfig.get_complete_url signature ) -> str: - resolved_base: Final = (api_base or get_secret_str("NIMBLE_API_BASE") or self.NIMBLE_API_BASE).rstrip("/") - if resolved_base.endswith("/search"): - return resolved_base - return f"{resolved_base}/search" + return ensure_api_route(api_base or get_secret_str("NIMBLE_API_BASE") or self.NIMBLE_API_BASE, "search") def transform_search_request( self, diff --git a/litellm/llms/nvidia_nim/rerank/ranking_transformation.py b/litellm/llms/nvidia_nim/rerank/ranking_transformation.py index 177d7883feb..39391be0fdb 100644 --- a/litellm/llms/nvidia_nim/rerank/ranking_transformation.py +++ b/litellm/llms/nvidia_nim/rerank/ranking_transformation.py @@ -11,6 +11,7 @@ from typing import Final import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.nvidia_nim.rerank.transformation import NvidiaNimRerankConfig from litellm.types.rerank import RerankResponse @@ -76,7 +77,7 @@ class NvidiaNimRankingConfig(NvidiaNimRerankConfig): api_base = api_base.removesuffix("/v1") - return f"{api_base}/v1/ranking" + return join_api_base(api_base, "v1/ranking") def map_cohere_rerank_params( self, diff --git a/litellm/llms/nvidia_nim/rerank/transformation.py b/litellm/llms/nvidia_nim/rerank/transformation.py index c8e1a331c8c..12377d8f854 100644 --- a/litellm/llms/nvidia_nim/rerank/transformation.py +++ b/litellm/llms/nvidia_nim/rerank/transformation.py @@ -8,6 +8,7 @@ from typing_extensions import Required, TypedDict import litellm from litellm._uuid import uuid from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str @@ -106,7 +107,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig): # Strip nvidia_nim/ prefix from model name if present clean_model: Final = self._get_clean_model_name(model) - return f"{api_base}/v1/retrieval/{clean_model}/reranking" + return join_api_base(api_base, f"v1/retrieval/{clean_model}/reranking") def get_supported_cohere_rerank_params(self, model: str) -> list: """ diff --git a/litellm/llms/oci/chat/transformation.py b/litellm/llms/oci/chat/transformation.py index e62e5e63642..8e1502b2610 100644 --- a/litellm/llms/oci/chat/transformation.py +++ b/litellm/llms/oci/chat/transformation.py @@ -21,6 +21,7 @@ from litellm._logging import verbose_logger from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS from litellm.litellm_core_utils.logging_utils import track_llm_api_timing from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, @@ -538,7 +539,7 @@ class OCIChatConfig(BaseConfig): stream: bool | None = None, ) -> str: base: Final = get_oci_base_url(optional_params, api_base or litellm.api_base) - return f"{base}/{OCI_API_VERSION}/actions/chat" + return join_api_base(base, f"{OCI_API_VERSION}/actions/chat") def _get_optional_params(self, vendor: OCIVendors, optional_params: dict, model: str = "") -> dict: param_map: Final = ( diff --git a/litellm/llms/oci/embed/transformation.py b/litellm/llms/oci/embed/transformation.py index edadc293771..ffcb156bb16 100644 --- a/litellm/llms/oci/embed/transformation.py +++ b/litellm/llms/oci/embed/transformation.py @@ -29,6 +29,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.oci.common_utils import ( @@ -156,7 +157,7 @@ class OCIEmbedConfig(BaseEmbeddingConfig): stream: bool | None = None, ) -> str: base: Final = get_oci_base_url(optional_params, api_base or litellm.api_base) - return f"{base}/{OCI_API_VERSION}/actions/embedText" + return join_api_base(base, f"{OCI_API_VERSION}/actions/embedText") def sign_request( self, diff --git a/litellm/llms/ollama/chat/transformation.py b/litellm/llms/ollama/chat/transformation.py index 644ab187f15..f9e7071db41 100644 --- a/litellm/llms/ollama/chat/transformation.py +++ b/litellm/llms/ollama/chat/transformation.py @@ -14,6 +14,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_images_from_message, extract_reasoning_content, ) +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.ollama import ( @@ -218,10 +219,7 @@ class OllamaChatConfig(BaseConfig): Some providers need `model` in `api_base` """ - clean_api_base: Final = (api_base or "http://localhost:11434").rstrip("/") - if clean_api_base.endswith("/api/chat"): - return clean_api_base - return f"{clean_api_base}/api/chat" + return ensure_api_route(api_base or "http://localhost:11434", "api/chat") def transform_request( self, diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 284a6ffea62..7894904c870 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -8,6 +8,7 @@ from pydantic import BaseModel, TypeAdapter from litellm import verbose_logger from litellm._uuid import uuid +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.utils import ModelResponse, ModelResponseStream @@ -149,7 +150,7 @@ class OllamaModelInfo(BaseLLMModelInfo): names: Final[set[str]] = set() try: - resp: Final = httpx.get(f"{base}/api/tags", headers=headers) + resp: Final = httpx.get(join_api_base(base, "api/tags"), headers=headers) resp.raise_for_status() data: Final = resp.json() # Expecting a dict with a 'models' list @@ -230,7 +231,7 @@ class OllamaModelInfo(BaseLLMModelInfo): try: response: Final = module_level_client.post( - url=f"{api_base}/api/show", + url=join_api_base(api_base, "api/show"), json={"name": model}, headers=headers, ) diff --git a/litellm/llms/ollama/completion/handler.py b/litellm/llms/ollama/completion/handler.py index 816028499e8..d4110e55bf0 100644 --- a/litellm/llms/ollama/completion/handler.py +++ b/litellm/llms/ollama/completion/handler.py @@ -10,6 +10,7 @@ from typing import Any, Final, Protocol, TypedDict from typing_extensions import NotRequired, ReadOnly import litellm +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.types.utils import EmbeddingResponse @@ -26,13 +27,6 @@ class OllamaEmbeddingResponse(TypedDict): prompt_eval_count: ReadOnly[NotRequired[int]] -def _ollama_embed_url(api_base: str) -> str: - clean_api_base: Final = api_base.rstrip("/") - if clean_api_base.endswith("/api/embed"): - return clean_api_base - return f"{clean_api_base}/api/embed" - - def _prepare_ollama_embedding_payload( model: str, prompts: list[str], optional_params: Mapping[str, object] ) -> dict[str, object]: @@ -99,7 +93,7 @@ async def ollama_aembeddings( ): data: Final = _prepare_ollama_embedding_payload(model, prompts, optional_params) - response: Final = await litellm.module_level_aclient.post(url=_ollama_embed_url(api_base), json=data) + response: Final = await litellm.module_level_aclient.post(url=ensure_api_route(api_base, "api/embed"), json=data) response_json: Final[OllamaEmbeddingResponse] = response.json() return _process_ollama_embedding_response( @@ -123,7 +117,7 @@ def ollama_embeddings( ): data: Final = _prepare_ollama_embedding_payload(model, prompts, optional_params) - response: Final = litellm.module_level_client.post(url=_ollama_embed_url(api_base), json=data) + response: Final = litellm.module_level_client.post(url=ensure_api_route(api_base, "api/embed"), json=data) response_json: Final[OllamaEmbeddingResponse] = response.json() return _process_ollama_embedding_response( diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index ac41310fb61..30de22b17ff 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -23,6 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_inline_remote_media, inline_remote_image_urls, ) +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.base import LiteLLMBaseModel @@ -500,10 +501,7 @@ class OllamaConfig(BaseConfig): Some providers need `model` in `api_base` """ - clean_api_base: Final = (api_base or "http://localhost:11434").rstrip("/") - if clean_api_base.endswith("/api/generate"): - return clean_api_base - return f"{clean_api_base}/api/generate" + return ensure_api_route(api_base or "http://localhost:11434", "api/generate") def get_model_response_iterator( self, diff --git a/litellm/llms/oobabooga/chat/oobabooga.py b/litellm/llms/oobabooga/chat/oobabooga.py index e21a165d757..8879c20c5eb 100644 --- a/litellm/llms/oobabooga/chat/oobabooga.py +++ b/litellm/llms/oobabooga/chat/oobabooga.py @@ -3,6 +3,7 @@ from collections.abc import Callable from typing import TYPE_CHECKING, Final import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import get_httpx_client from litellm.utils import EmbeddingResponse, ModelResponse, Usage @@ -49,7 +50,7 @@ def completion( ) model = model - completion_url = completion_url + "/v1/chat/completions" + completion_url = join_api_base(completion_url, "v1/chat/completions") data: Final = oobabooga_config.transform_request( model=model, messages=messages, @@ -103,7 +104,7 @@ def embedding( if model.startswith(("http://", "https://")): embeddings_url = model elif api_base: - embeddings_url = f"{api_base}/v1/embeddings" + embeddings_url = join_api_base(api_base, "v1/embeddings") else: raise OobaboogaError( status_code=404, diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py index 4d7ef833181..b4c92848752 100644 --- a/litellm/llms/openai/chat/gpt_transformation.py +++ b/litellm/llms/openai/chat/gpt_transformation.py @@ -34,6 +34,7 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import ( async_convert_url_to_base64, convert_url_to_base64, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException @@ -807,7 +808,7 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig): if endpoint in api_base: return api_base - return f"{api_base}/{endpoint}" + return join_api_base(api_base, endpoint) def validate_environment( self, diff --git a/litellm/llms/openai/containers/transformation.py b/litellm/llms/openai/containers/transformation.py index 6cb2396b85d..5d03585189d 100644 --- a/litellm/llms/openai/containers/transformation.py +++ b/litellm/llms/openai/containers/transformation.py @@ -9,7 +9,7 @@ from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.containers.main import ( ContainerCreateOptionalRequestParams, @@ -130,7 +130,7 @@ class OpenAIContainerConfig(BaseContainerConfig): or "https://api.openai.com/v1" ) - return f"{api_base.rstrip('/')}/containers" + return join_api_base(api_base, "containers") def transform_container_create_request( self, diff --git a/litellm/llms/openai/evals/transformation.py b/litellm/llms/openai/evals/transformation.py index 24de7f6d479..581e5e8ad87 100644 --- a/litellm/llms/openai/evals/transformation.py +++ b/litellm/llms/openai/evals/transformation.py @@ -8,7 +8,7 @@ from typing import Final import httpx from litellm._logging import verbose_logger -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.evals.transformation import ( BaseEvalsAPIConfig, LiteLLMLoggingObj, @@ -67,8 +67,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): if eval_id: encoded_eval_id: Final = encode_url_path_segment(eval_id, field_name="eval_id") - return f"{api_base}/v1/evals/{encoded_eval_id}" - return f"{api_base}/v1/{endpoint}" + return join_api_base(api_base, f"v1/evals/{encoded_eval_id}") + return join_api_base(api_base, f"v1/{endpoint}") def transform_create_eval_request( self, @@ -226,7 +226,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): headers: dict, ) -> tuple[str, dict, dict]: """Transform cancel eval request for OpenAI""" - url: Final = f"{self.get_complete_url(api_base=api_base, endpoint='evals', eval_id=eval_id)}/cancel" + url: Final = join_api_base( + self.get_complete_url(api_base=api_base, endpoint="evals", eval_id=eval_id), "cancel" + ) # Empty body for cancel request request_body: Final[dict[str, object]] = {} @@ -260,7 +262,7 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): api_base = litellm_params.api_base encoded_eval_id: Final = encode_url_path_segment(eval_id, field_name="eval_id") - url: Final = f"{api_base}/v1/evals/{encoded_eval_id}/runs" + url: Final = join_api_base(api_base, f"v1/evals/{encoded_eval_id}/runs") # Build request body request_body: Final = {k: v for k, v in create_request.items() if v is not None} @@ -293,7 +295,7 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): api_base = litellm_params.api_base encoded_eval_id: Final = encode_url_path_segment(eval_id, field_name="eval_id") - url: Final = f"{api_base}/v1/evals/{encoded_eval_id}/runs" + url: Final = join_api_base(api_base, f"v1/evals/{encoded_eval_id}/runs") # Build query parameters query_params: Final[dict[str, object]] = {} @@ -335,7 +337,7 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): """Transform get run request for OpenAI""" encoded_eval_id: Final = encode_url_path_segment(eval_id, field_name="eval_id") encoded_run_id: Final = encode_url_path_segment(run_id, field_name="run_id") - url: Final = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}" + url: Final = join_api_base(api_base, f"v1/evals/{encoded_eval_id}/runs/{encoded_run_id}") verbose_logger.debug("Get run request - URL: %s", url) @@ -363,7 +365,7 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): """Transform cancel run request for OpenAI""" encoded_eval_id: Final = encode_url_path_segment(eval_id, field_name="eval_id") encoded_run_id: Final = encode_url_path_segment(run_id, field_name="run_id") - url: Final = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}/cancel" + url: Final = join_api_base(api_base, f"v1/evals/{encoded_eval_id}/runs/{encoded_run_id}/cancel") # Empty body for cancel request request_body: Final[dict[str, object]] = {} @@ -394,7 +396,7 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig): """Transform delete run request for OpenAI""" encoded_eval_id: Final = encode_url_path_segment(eval_id, field_name="eval_id") encoded_run_id: Final = encode_url_path_segment(run_id, field_name="run_id") - url: Final = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}" + url: Final = join_api_base(api_base, f"v1/evals/{encoded_eval_id}/runs/{encoded_run_id}") # Empty body for delete request request_body: Final[dict[str, object]] = {} diff --git a/litellm/llms/openai/image_edit/transformation.py b/litellm/llms/openai/image_edit/transformation.py index a9e5b1b9f34..9def833052b 100644 --- a/litellm/llms/openai/image_edit/transformation.py +++ b/litellm/llms/openai/image_edit/transformation.py @@ -6,6 +6,7 @@ from httpx._types import RequestFiles import litellm from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ( @@ -184,7 +185,4 @@ class OpenAIImageEditConfig(BaseImageEditConfig): or "https://api.openai.com/v1" ) - # Remove trailing slashes - api_base = api_base.rstrip("/") - - return f"{api_base}/images/edits" + return join_api_base(api_base, "images/edits") diff --git a/litellm/llms/openai/realtime/http_transformation.py b/litellm/llms/openai/realtime/http_transformation.py index 61dbf20397f..49d33582c6f 100644 --- a/litellm/llms/openai/realtime/http_transformation.py +++ b/litellm/llms/openai/realtime/http_transformation.py @@ -1,6 +1,7 @@ """OpenAI realtime HTTP transformation config (client_secrets + realtime_calls).""" import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPConfig from litellm.secret_managers.main import get_secret_str @@ -15,17 +16,17 @@ class OpenAIRealtimeHTTPConfig(BaseRealtimeHTTPConfig): def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base = self.get_api_base(api_base).rstrip("/") base = base.removesuffix("/v1") - return f"{base}/v1/realtime/client_secrets" + return join_api_base(base, "v1/realtime/client_secrets") def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base = self.get_api_base(api_base).rstrip("/") base = base.removesuffix("/v1") - return f"{base}/v1/realtime/calls" + return join_api_base(base, "v1/realtime/calls") def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str: base = self.get_api_base(api_base).rstrip("/") base = base.removesuffix("/v1") - return f"{base}/v1/realtime/transcription_sessions" + return join_api_base(base, "v1/realtime/transcription_sessions") def validate_environment( self, diff --git a/litellm/llms/openai/responses/count_tokens/transformation.py b/litellm/llms/openai/responses/count_tokens/transformation.py index 6d882e0eeb1..c1ee42444c5 100644 --- a/litellm/llms/openai/responses/count_tokens/transformation.py +++ b/litellm/llms/openai/responses/count_tokens/transformation.py @@ -13,6 +13,7 @@ from pydantic import JsonValue, TypeAdapter, ValidationError from typing_extensions import ReadOnly, TypedDict from litellm.litellm_core_utils.tool_fields import DropPolicy +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages from litellm.llms.anthropic.pass_through.responses_adapters.transformation import ( @@ -148,9 +149,8 @@ class OpenAICountTokensConfig: """ def get_openai_count_tokens_endpoint(self, api_base: str | None = None) -> str: - base = api_base or "https://api.openai.com/v1" - base = base.rstrip("/") - return f"{base}/responses/input_tokens" + base: Final = api_base or "https://api.openai.com/v1" + return join_api_base(base, "responses/input_tokens") def transform_request_to_count_tokens( self, diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 8f29a71d63b..b90f6aa01da 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -23,7 +23,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( with_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.safe_json_loads import safe_json_loads -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.openai.chat.gpt_5_transformation import ( is_gpt_reasoning_series_name, @@ -881,10 +881,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): or "https://api.openai.com/v1" ) - # Remove trailing slashes - api_base = api_base.rstrip("/") - - return f"{api_base}/responses" + return join_api_base(api_base, "responses") def transform_streaming_response( self, @@ -1111,7 +1108,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): order: Literal["asc", "desc"] = "desc", ) -> tuple[str, dict]: encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") - url: Final = f"{api_base}/{encoded_response_id}/input_items" + url: Final = join_api_base(api_base, f"{encoded_response_id}/input_items") params: Final[dict[str, object]] = {} if after is not None: params["after"] = after @@ -1152,7 +1149,7 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): - POST /v1/responses/{response_id}/cancel """ encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") - url: Final = f"{api_base}/{encoded_response_id}/cancel" + url: Final = join_api_base(api_base, f"{encoded_response_id}/cancel") data: Final[dict] = {} return url, data diff --git a/litellm/llms/openai/transcriptions/whisper_transformation.py b/litellm/llms/openai/transcriptions/whisper_transformation.py index 8f96f688189..eb4f1b6c9ce 100644 --- a/litellm/llms/openai/transcriptions/whisper_transformation.py +++ b/litellm/llms/openai/transcriptions/whisper_transformation.py @@ -4,6 +4,7 @@ from typing import Final from httpx import Headers, Response +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -46,9 +47,9 @@ class OpenAIWhisperAudioTranscriptionConfig(BaseAudioTranscriptionConfig): # if endswith "/v1" if api_base and api_base.endswith("/v1"): - api_base = f"{api_base}/audio/transcriptions" + api_base = join_api_base(api_base, "audio/transcriptions") else: - api_base = f"{api_base}/v1/audio/transcriptions" + api_base = join_api_base(api_base, "v1/audio/transcriptions") return api_base or "" diff --git a/litellm/llms/openai/vector_store_files/transformation.py b/litellm/llms/openai/vector_store_files/transformation.py index 7f65ade224e..bd8145bb460 100644 --- a/litellm/llms/openai/vector_store_files/transformation.py +++ b/litellm/llms/openai/vector_store_files/transformation.py @@ -4,7 +4,7 @@ from typing import Final, cast import httpx import litellm -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.vector_store_files.transformation import ( BaseVectorStoreFilesConfig, ) @@ -92,16 +92,15 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig): vector_store_id: str, litellm_params: Mapping[str, object], ) -> str: - base_url = ( + base_url: Final = ( api_base or litellm.api_base or get_secret_str("OPENAI_BASE_URL") or get_secret_str("OPENAI_API_BASE") or "https://api.openai.com/v1" ) - base_url = base_url.rstrip("/") encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - return f"{base_url}/vector_stores/{encoded_vector_store_id}/files" + return join_api_base(base_url, f"vector_stores/{encoded_vector_store_id}/files") def transform_create_vector_store_file_request( self, @@ -187,7 +186,7 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig): api_base: str, ) -> tuple[str, dict[str, object]]: encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id") - return f"{api_base}/{encoded_file_id}/content", {} + return join_api_base(api_base, f"{encoded_file_id}/content"), {} def transform_retrieve_vector_store_file_content_response( self, diff --git a/litellm/llms/openai/vector_stores/transformation.py b/litellm/llms/openai/vector_stores/transformation.py index 46105a66be3..d108802f7f9 100644 --- a/litellm/llms/openai/vector_stores/transformation.py +++ b/litellm/llms/openai/vector_stores/transformation.py @@ -3,7 +3,7 @@ from typing import TYPE_CHECKING, Any, Final, cast import httpx import litellm -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.openai.workload_identity import resolve_openai_bearer_token from litellm.secret_managers.main import get_secret_str @@ -88,10 +88,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): or "https://api.openai.com/v1" ) - # Remove trailing slashes - api_base = api_base.rstrip("/") - - return f"{api_base}/vector_stores" + return join_api_base(api_base, "vector_stores") def transform_search_vector_store_request( self, @@ -105,7 +102,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig): ) -> tuple[str, dict]: encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") base_url, query_separator, query_string = api_base.partition("?") - url: Final = f"{base_url}/{encoded_vector_store_id}/search{query_separator}{query_string}" + url: Final = f"{join_api_base(base_url, f'{encoded_vector_store_id}/search')}{query_separator}{query_string}" typed_request_body: Final = VectorStoreSearchRequest( query=query, filters=vector_store_search_optional_params.get("filters", None), diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index d422e51f649..0560e1c2e3a 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -8,7 +8,7 @@ from urllib.parse import quote import httpx from httpx._types import FileContent, FileTypes, RequestFiles -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils from litellm.llms.openai.workload_identity import resolve_openai_bearer_token @@ -98,7 +98,7 @@ class OpenAIVideoConfig(BaseVideoConfig): if api_base is None: api_base = "https://api.openai.com/v1" - return f"{api_base.rstrip('/')}/videos" + return join_api_base(api_base, "videos") def use_multipart_form_data(self) -> bool: return True @@ -214,7 +214,7 @@ class OpenAIVideoConfig(BaseVideoConfig): encoded_video_id: Final = encode_url_path_segment(original_video_id, field_name="video_id") # Construct the URL for video content download - url = f"{api_base.rstrip('/')}/{encoded_video_id}/content" + url = join_api_base(api_base, f"{encoded_video_id}/content") if variant is not None: # Encode the user-controlled ``variant`` so a value like # ``thumbnail&extra=1`` cannot inject additional query params @@ -246,7 +246,7 @@ class OpenAIVideoConfig(BaseVideoConfig): encoded_video_id: Final = encode_url_path_segment(original_video_id, field_name="video_id") # Construct the URL for video remix - url: Final = f"{api_base.rstrip('/')}/{encoded_video_id}/remix" + url: Final = join_api_base(api_base, f"{encoded_video_id}/remix") # Prepare the request data data: Final[dict[str, object]] = {"prompt": prompt} @@ -465,7 +465,7 @@ class OpenAIVideoConfig(BaseVideoConfig): litellm_params: GenericLiteLLMParams, headers: dict, ) -> tuple[str, list]: - url: Final = f"{api_base.rstrip('/')}/characters" + url: Final = join_api_base(api_base, "characters") files_list: Final[list[tuple[str, FileTypes]]] = [("name", (None, name))] self._add_video_to_files(files_list, video, "video") return url, files_list @@ -486,7 +486,7 @@ class OpenAIVideoConfig(BaseVideoConfig): ) -> tuple[str, dict]: original_character_id: Final = extract_original_character_id(character_id) encoded_character_id: Final = encode_url_path_segment(original_character_id, field_name="character_id") - url: Final = f"{api_base.rstrip('/')}/characters/{encoded_character_id}" + url: Final = join_api_base(api_base, f"characters/{encoded_character_id}") return url, {} def transform_video_get_character_response( @@ -507,7 +507,7 @@ class OpenAIVideoConfig(BaseVideoConfig): extra_body: dict[str, object] | None = None, prefetched_source_data: dict[str, object] | None = None, ) -> tuple[str, Mapping[str, object], RequestFiles | None]: - url: Final = f"{api_base.rstrip('/')}/edits" + url: Final = join_api_base(api_base, "edits") if video_file is not None: files: Final[RequestFiles] = (self._video_file_tuple(video_file, "video"),) @@ -547,7 +547,7 @@ class OpenAIVideoConfig(BaseVideoConfig): extra_body: dict[str, object] | None = None, ) -> tuple[str, dict]: original_video_id: Final = extract_original_video_id(video_id) - url: Final = f"{api_base.rstrip('/')}/extensions" + url: Final = join_api_base(api_base, "extensions") data: Final[dict[str, object]] = { "prompt": prompt, "seconds": seconds, diff --git a/litellm/llms/openai_like/common_utils.py b/litellm/llms/openai_like/common_utils.py index 11b85e52af5..5d3085a3901 100644 --- a/litellm/llms/openai_like/common_utils.py +++ b/litellm/llms/openai_like/common_utils.py @@ -2,6 +2,8 @@ from typing import Literal import httpx +from litellm.litellm_core_utils.url_utils import join_api_base + class OpenAILikeError(Exception): def __init__(self, status_code, message): @@ -48,7 +50,7 @@ class OpenAILikeBase: if not custom_endpoint: if endpoint_type == "chat_completions": - api_base = f"{api_base}/chat/completions" + api_base = join_api_base(api_base, "chat/completions") elif endpoint_type == "embeddings": - api_base = f"{api_base}/embeddings" + api_base = join_api_base(api_base, "embeddings") return api_base, headers diff --git a/litellm/llms/openai_like/dynamic_config.py b/litellm/llms/openai_like/dynamic_config.py index f0bf710b951..685561f55a5 100644 --- a/litellm/llms/openai_like/dynamic_config.py +++ b/litellm/llms/openai_like/dynamic_config.py @@ -9,6 +9,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( handle_messages_with_content_list_to_str_conversion, ) +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig from litellm.llms.openai_like.chat.transformation import OpenAILikeChatConfig from litellm.secret_managers.main import get_secret_str @@ -91,13 +92,9 @@ def create_config_class(provider: SimpleProviderConfig): if not api_base: api_base = provider.base_url - if api_base is None: - raise ValueError(f"api_base is required for provider {provider.slug}") - - if not api_base.endswith("/chat/completions"): - api_base = f"{api_base}/chat/completions" - - return api_base + if isinstance(api_base, str): + return ensure_api_route(api_base, "chat/completions") + raise ValueError(f"api_base is required for provider {provider.slug}") def get_supported_openai_params(self, model: str) -> list: """Get supported OpenAI params, excluding tool-related params for models @@ -226,11 +223,9 @@ def create_responses_config_class(provider: SimpleProviderConfig) -> "type[OpenA if not api_base: api_base = provider.base_url - if api_base is None: - raise ValueError(f"api_base is required for provider {provider.slug}") - - api_base = api_base.rstrip("/") - return f"{api_base}/responses" + if isinstance(api_base, str): + return join_api_base(api_base, "responses") + raise ValueError(f"api_base is required for provider {provider.slug}") def transform_responses_api_request( self, diff --git a/litellm/llms/openai_like/messages/transformation.py b/litellm/llms/openai_like/messages/transformation.py index ef501720e2d..12c8b1da166 100644 --- a/litellm/llms/openai_like/messages/transformation.py +++ b/litellm/llms/openai_like/messages/transformation.py @@ -1,6 +1,7 @@ from typing import Any, Final import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, @@ -113,7 +114,7 @@ class OpenAILikeAnthropicMessagesConfig(AnthropicMessagesConfig): if base.endswith("/v1/messages"): return base base = base.removesuffix("/v1") - return f"{base}/v1/messages" + return join_api_base(base, "v1/messages") class JSONProviderAnthropicMessagesConfig(OpenAILikeAnthropicMessagesConfig): diff --git a/litellm/llms/openai_like/responses/transformation.py b/litellm/llms/openai_like/responses/transformation.py index 9655772e953..68de4065b09 100644 --- a/litellm/llms/openai_like/responses/transformation.py +++ b/litellm/llms/openai_like/responses/transformation.py @@ -8,6 +8,7 @@ provider-specific resolution (slug, API key env var, base URL). from typing import Final +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams @@ -47,5 +48,4 @@ class OpenAILikeResponsesConfig(OpenAIResponsesAPIConfig): api_base = api_base or get_secret_str("OPENAI_LIKE_API_BASE") if not api_base: raise ValueError("api_base is required for openai_like provider") - api_base = api_base.rstrip("/") - return f"{api_base}/responses" + return join_api_base(api_base, "responses") diff --git a/litellm/llms/openrouter/embedding/transformation.py b/litellm/llms/openrouter/embedding/transformation.py index 14b0e462ea7..d1530a29515 100644 --- a/litellm/llms/openrouter/embedding/transformation.py +++ b/litellm/llms/openrouter/embedding/transformation.py @@ -11,6 +11,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.types.llms.openai import AllEmbeddingInputValues from litellm.types.utils import EmbeddingResponse @@ -84,15 +85,7 @@ class OpenrouterEmbeddingConfig(BaseEmbeddingConfig): """ Get the complete URL for OpenRouter Embedding API endpoint. """ - # api_base is already set to https://openrouter.ai/api/v1 in main.py - # Remove trailing slashes - if api_base: - api_base = api_base.rstrip("/") - else: - api_base = "https://openrouter.ai/api/v1" - - # Return the embeddings endpoint - return f"{api_base}/embeddings" + return join_api_base(api_base or "https://openrouter.ai/api/v1", "embeddings") def transform_embedding_request( self, diff --git a/litellm/llms/openrouter/image_edit/transformation.py b/litellm/llms/openrouter/image_edit/transformation.py index a73a2fe4192..9644ee3f280 100644 --- a/litellm/llms/openrouter/image_edit/transformation.py +++ b/litellm/llms/openrouter/image_edit/transformation.py @@ -52,6 +52,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.openrouter.common_utils import OpenRouterException @@ -187,11 +188,8 @@ class OpenRouterImageEditConfig(BaseImageEditConfig): api_base: str | None, litellm_params: dict, ) -> str: - base_url = api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" - base_url = base_url.rstrip("/") - if not base_url.endswith("/chat/completions"): - return f"{base_url}/chat/completions" - return base_url + base_url: Final = api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" + return ensure_api_route(base_url, "chat/completions") def transform_image_edit_request( self, diff --git a/litellm/llms/openrouter/image_generation/transformation.py b/litellm/llms/openrouter/image_generation/transformation.py index 355df81b87a..ce6e62aaa88 100644 --- a/litellm/llms/openrouter/image_generation/transformation.py +++ b/litellm/llms/openrouter/image_generation/transformation.py @@ -35,6 +35,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, @@ -269,10 +270,7 @@ class OpenRouterImageGenerationConfig(BaseImageGenerationConfig): Default: https://openrouter.ai/api/v1/chat/completions """ if api_base: - if not api_base.endswith("/chat/completions"): - api_base = api_base.rstrip("/") - return f"{api_base}/chat/completions" - return api_base + return ensure_api_route(api_base, "chat/completions") return "https://openrouter.ai/api/v1/chat/completions" diff --git a/litellm/llms/openrouter/responses/transformation.py b/litellm/llms/openrouter/responses/transformation.py index aeb5b09e14a..e166ccaad2d 100644 --- a/litellm/llms/openrouter/responses/transformation.py +++ b/litellm/llms/openrouter/responses/transformation.py @@ -18,6 +18,7 @@ import litellm from litellm.litellm_core_utils.core_helpers import RESPONSE_COST_HEADER from litellm.litellm_core_utils.hidden_params import set_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -138,9 +139,7 @@ class OpenRouterResponsesAPIConfig(OpenAIResponsesAPIConfig): api_base or litellm.api_base or get_secret_str("OPENROUTER_API_BASE") or "https://openrouter.ai/api/v1" ) - api_base = api_base.rstrip("/") - - return f"{api_base}/responses" + return join_api_base(api_base, "responses") def supports_native_websocket(self) -> bool: """OpenRouter does not support native WebSocket for Responses API""" diff --git a/litellm/llms/opensandbox/sandbox/transformation.py b/litellm/llms/opensandbox/sandbox/transformation.py index 13c9d805638..574fbfa9e5b 100644 --- a/litellm/llms/opensandbox/sandbox/transformation.py +++ b/litellm/llms/opensandbox/sandbox/transformation.py @@ -21,6 +21,7 @@ from litellm.constants import ( OPEN_SANDBOX_READY_TIMEOUT, ) from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.sandbox.transformation import ( SANDBOX_MAX_OUTPUT_BYTES, BaseSandboxConfig, @@ -93,7 +94,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): ) response: Final = await self._http(client).post( - url=f"{base}/sandboxes", + url=join_api_base(base, "sandboxes"), headers=self._lifecycle_headers(key), json=body, ) @@ -192,7 +193,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): key: Final = self._api_key(api_key=api_key, handle=handle) try: response: Final = await self._http(client).delete( - url=f"{base}/sandboxes/{handle.id}", + url=join_api_base(base, f"sandboxes/{handle.id}"), headers=self._lifecycle_headers(key), ) except httpx.HTTPStatusError as e: @@ -258,7 +259,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): deadline: Final = time.monotonic() + ready_timeout while True: response = await self._http(client).get( - url=f"{api_base}/sandboxes/{sandbox_id}", + url=join_api_base(api_base, f"sandboxes/{sandbox_id}"), headers=headers, ) data = _JSON_VALUE.validate_python(response.json()) @@ -316,7 +317,7 @@ class OpenSandboxSandboxConfig(BaseSandboxConfig): client: AsyncHTTPHandler | None, ) -> tuple[str, dict[str, str]]: response: Final = await self._http(client).get( - url=f"{api_base}/sandboxes/{sandbox_id}/endpoints/{OPEN_SANDBOX_EXECD_PORT}", + url=join_api_base(api_base, f"sandboxes/{sandbox_id}/endpoints/{OPEN_SANDBOX_EXECD_PORT}"), headers=headers, params={"use_server_proxy": use_server_proxy}, ) diff --git a/litellm/llms/ovhcloud/audio_transcription/transformation.py b/litellm/llms/ovhcloud/audio_transcription/transformation.py index 3ae5f6fa69c..dd61bfb0c13 100644 --- a/litellm/llms/ovhcloud/audio_transcription/transformation.py +++ b/litellm/llms/ovhcloud/audio_transcription/transformation.py @@ -12,6 +12,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -63,9 +64,8 @@ class OVHCloudAudioTranscriptionConfig(BaseAudioTranscriptionConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - api_base = "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" if api_base is None else api_base.rstrip("/") - complete_url: Final = f"{api_base}/audio/transcriptions" - return complete_url + base_url: Final = "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" if api_base is None else api_base + return join_api_base(base_url, "audio/transcriptions") def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return OVHCloudException( diff --git a/litellm/llms/ovhcloud/chat/transformation.py b/litellm/llms/ovhcloud/chat/transformation.py index 3d3953eed34..91b3dbe6268 100644 --- a/litellm/llms/ovhcloud/chat/transformation.py +++ b/litellm/llms/ovhcloud/chat/transformation.py @@ -9,6 +9,7 @@ from typing import Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig @@ -31,9 +32,8 @@ class OVHCloudChatConfig(OpenAIGPTConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - api_base = "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" if api_base is None else api_base.rstrip("/") - complete_url: Final = f"{api_base}/chat/completions" - return complete_url + base_url: Final = "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" if api_base is None else api_base + return join_api_base(base_url, "chat/completions") def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return OVHCloudException( diff --git a/litellm/llms/ovhcloud/embedding/transformation.py b/litellm/llms/ovhcloud/embedding/transformation.py index cdf095e6ad1..487c40afa52 100644 --- a/litellm/llms/ovhcloud/embedding/transformation.py +++ b/litellm/llms/ovhcloud/embedding/transformation.py @@ -10,6 +10,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str @@ -35,9 +36,8 @@ class OVHCloudEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - api_base = "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" if api_base is None else api_base.rstrip("/") - complete_url: Final = f"{api_base}/embeddings" - return complete_url + base_url: Final = "https://oai.endpoints.kepler.ai.cloud.ovh.net/v1" if api_base is None else api_base + return join_api_base(base_url, "embeddings") def validate_environment( self, diff --git a/litellm/llms/perplexity/embedding/transformation.py b/litellm/llms/perplexity/embedding/transformation.py index 945a58d4d75..7b4da933976 100644 --- a/litellm/llms/perplexity/embedding/transformation.py +++ b/litellm/llms/perplexity/embedding/transformation.py @@ -20,6 +20,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str @@ -68,7 +69,7 @@ class PerplexityEmbeddingConfig(BaseEmbeddingConfig): if api_base: clean_api_base: Final = api_base.rstrip("/") if not clean_api_base.endswith("/embeddings"): - return f"{clean_api_base}/v1/embeddings" + return join_api_base(clean_api_base, "v1/embeddings") return clean_api_base return "https://api.perplexity.ai/v1/embeddings" diff --git a/litellm/llms/perplexity/responses/transformation.py b/litellm/llms/perplexity/responses/transformation.py index 91d40d8247f..a2b7eceea40 100644 --- a/litellm/llms/perplexity/responses/transformation.py +++ b/litellm/llms/perplexity/responses/transformation.py @@ -14,6 +14,7 @@ from typing import Any, Final import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str @@ -51,7 +52,7 @@ class PerplexityResponsesConfig(OpenAIResponsesAPIConfig): def get_complete_url(self, api_base: str | None, litellm_params: dict) -> str: api_base = api_base or get_secret_str("PERPLEXITY_API_BASE") or "https://api.perplexity.ai" - return f"{api_base.rstrip('/')}/v1/responses" + return join_api_base(api_base, "v1/responses") def _ensure_message_type(self, input: str | ResponseInputParam) -> str | ResponseInputParam: """Ensure list input items have type='message' (required by Perplexity).""" diff --git a/litellm/llms/perplexity/search/transformation.py b/litellm/llms/perplexity/search/transformation.py index b46e4225309..1f3b0fe729d 100644 --- a/litellm/llms/perplexity/search/transformation.py +++ b/litellm/llms/perplexity/search/transformation.py @@ -9,6 +9,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -78,11 +79,7 @@ class PerplexitySearchConfig(BaseSearchConfig): """ Get complete URL for Search endpoint. """ - resolved_api_base: Final = api_base or get_secret_str("PERPLEXITY_API_BASE") or self.PERPLEXITY_API_BASE - clean_api_base: Final = resolved_api_base.rstrip("/") - if clean_api_base.endswith("/search"): - return clean_api_base - return f"{clean_api_base}/search" + return ensure_api_route(api_base or get_secret_str("PERPLEXITY_API_BASE") or self.PERPLEXITY_API_BASE, "search") def transform_search_request( self, diff --git a/litellm/llms/pg_vector/vector_stores/transformation.py b/litellm/llms/pg_vector/vector_stores/transformation.py index e4b06c36bf4..eeecd02653d 100644 --- a/litellm/llms/pg_vector/vector_stores/transformation.py +++ b/litellm/llms/pg_vector/vector_stores/transformation.py @@ -1,6 +1,6 @@ from typing import TYPE_CHECKING, Any, Final -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams @@ -66,10 +66,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): "PG Vector API base URL is required. Set PG_VECTOR_API_BASE environment variable or pass api_base in litellm_params." ) - # Remove trailing slashes - api_base = api_base.rstrip("/") - - return f"{api_base}/v1/vector_stores" + return join_api_base(api_base, "v1/vector_stores") def transform_search_vector_store_request( self, @@ -82,7 +79,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig): extra_body: dict[str, Any] | None = None, ) -> tuple[str, dict]: encoded_vector_store_id: Final = encode_url_path_segment(vector_store_id, field_name="vector_store_id") - url: Final = f"{api_base}/{encoded_vector_store_id}/search" + url: Final = join_api_base(api_base, f"{encoded_vector_store_id}/search") _, request_body = super().transform_search_vector_store_request( vector_store_id=vector_store_id, query=query, diff --git a/litellm/llms/predibase/chat/transformation.py b/litellm/llms/predibase/chat/transformation.py index 0dd7e27722e..b30713fa119 100644 --- a/litellm/llms/predibase/chat/transformation.py +++ b/litellm/llms/predibase/chat/transformation.py @@ -13,6 +13,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( prompt_factory, render_custom_prompt_template, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import Choices, Message, ModelResponse, Usage @@ -337,12 +338,12 @@ class PredibaseConfig(BaseConfig): elif "PREDIBASE_API_BASE" in os.environ: base_url = os.getenv("PREDIBASE_API_BASE", "") - completion_url = f"{base_url}/{tenant_id}/deployments/v2/llms/{model}" + completion_url = join_api_base(base_url, f"{tenant_id}/deployments/v2/llms/{model}") should_stream: Final = stream if stream is not None else optional_params.get("stream", False) if should_stream is True: - completion_url += "/generate_stream" + completion_url = join_api_base(completion_url, "generate_stream") else: - completion_url += "/generate" + completion_url = join_api_base(completion_url, "generate") return completion_url def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException: diff --git a/litellm/llms/ragflow/chat/transformation.py b/litellm/llms/ragflow/chat/transformation.py index 97b723f7dd5..f626cd8e55f 100644 --- a/litellm/llms/ragflow/chat/transformation.py +++ b/litellm/llms/ragflow/chat/transformation.py @@ -13,7 +13,7 @@ Model name format: from typing import Final import litellm -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.openai.openai import OpenAIConfig from litellm.secret_managers.main import get_secret, get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -122,7 +122,7 @@ class RAGFlowConfig(OpenAIConfig): if not path.startswith("/"): path = "/" + path - return f"{api_base}{path}" + return join_api_base(api_base, path) def _get_openai_compatible_provider_info( self, diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py index 007b047cc1b..ef44b6e4c0f 100644 --- a/litellm/llms/ragflow/vector_stores/transformation.py +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -5,6 +5,7 @@ import httpx from pydantic import ConfigDict, RootModel, with_config from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams @@ -95,15 +96,16 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): - api_base in litellm_params - Default: http://localhost:9380 """ - api_base = ( - api_base or litellm_params.get("api_base") or get_secret_str("RAGFLOW_API_BASE") or "http://localhost:9380" + params_api_base: Final = litellm_params.get("api_base") + resolved_api_base: Final = ( + api_base + or (f"{params_api_base}" if params_api_base else None) + or get_secret_str("RAGFLOW_API_BASE") + or "http://localhost:9380" ) - # Remove trailing slashes - api_base = api_base.rstrip("/") - # RAGFlow datasets API endpoint - return f"{api_base}/api/v1/datasets" + return join_api_base(resolved_api_base, "api/v1/datasets") def transform_search_vector_store_request( self, diff --git a/litellm/llms/recraft/image_edit/transformation.py b/litellm/llms/recraft/image_edit/transformation.py index 46905580382..74bfa52c5e0 100644 --- a/litellm/llms/recraft/image_edit/transformation.py +++ b/litellm/llms/recraft/image_edit/transformation.py @@ -5,6 +5,7 @@ import httpx from httpx._types import RequestFiles from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams @@ -68,11 +69,8 @@ class RecraftImageEditConfig(BaseImageEditConfig): Some providers need `model` in `api_base` """ - complete_url: str = api_base or get_secret_str("RECRAFT_API_BASE") or self.DEFAULT_BASE_URL - - complete_url = complete_url.rstrip("/") - complete_url = f"{complete_url}/{self.IMAGE_EDIT_ENDPOINT}" - return complete_url + base_url: Final = api_base or get_secret_str("RECRAFT_API_BASE") or self.DEFAULT_BASE_URL + return join_api_base(base_url, self.IMAGE_EDIT_ENDPOINT) def validate_environment( self, diff --git a/litellm/llms/recraft/image_generation/transformation.py b/litellm/llms/recraft/image_generation/transformation.py index 0fcfafb79fb..7b1a7714ec6 100644 --- a/litellm/llms/recraft/image_generation/transformation.py +++ b/litellm/llms/recraft/image_generation/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -72,11 +73,8 @@ class RecraftImageGenerationConfig(BaseImageGenerationConfig): Some providers need `model` in `api_base` """ - complete_url: str = api_base or get_secret_str("RECRAFT_API_BASE") or self.DEFAULT_BASE_URL - - complete_url = complete_url.rstrip("/") - complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" - return complete_url + base_url: Final = api_base or get_secret_str("RECRAFT_API_BASE") or self.DEFAULT_BASE_URL + return join_api_base(base_url, self.IMAGE_GENERATION_ENDPOINT) def validate_environment( self, diff --git a/litellm/llms/runwayml/image_generation/transformation.py b/litellm/llms/runwayml/image_generation/transformation.py index ddc1b9474ac..e721ac678a1 100644 --- a/litellm/llms/runwayml/image_generation/transformation.py +++ b/litellm/llms/runwayml/image_generation/transformation.py @@ -11,6 +11,7 @@ from litellm.constants import ( RUNWAYML_DEFAULT_API_VERSION, RUNWAYML_POLLING_TIMEOUT, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -62,12 +63,8 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): Some providers need `model` in `api_base` """ - complete_url: str = api_base or get_secret_str("RUNWAYML_API_BASE") or self.DEFAULT_BASE_URL - - complete_url = complete_url.rstrip("/") - if self.IMAGE_GENERATION_ENDPOINT: - complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" - return complete_url + base_url: Final = api_base or get_secret_str("RUNWAYML_API_BASE") or self.DEFAULT_BASE_URL + return join_api_base(base_url, self.IMAGE_GENERATION_ENDPOINT) def validate_environment( self, @@ -226,8 +223,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): start_time: Final = time.time() # Build task status URL - api_base = api_base.rstrip("/") - task_url: Final = f"{api_base}/v1/tasks/{task_id}" + task_url: Final = join_api_base(api_base, f"v1/tasks/{task_id}") verbose_logger.debug("Polling RunwayML task: %s", task_url) @@ -275,8 +271,7 @@ class RunwayMLImageGenerationConfig(BaseImageGenerationConfig): start_time: Final = time.time() # Build task status URL - api_base = api_base.rstrip("/") - task_url: Final = f"{api_base}/v1/tasks/{task_id}" + task_url: Final = join_api_base(api_base, f"v1/tasks/{task_id}") verbose_logger.debug("Polling RunwayML task (async): %s", task_url) diff --git a/litellm/llms/runwayml/text_to_speech/transformation.py b/litellm/llms/runwayml/text_to_speech/transformation.py index d41ffea17dc..1f052648f13 100644 --- a/litellm/llms/runwayml/text_to_speech/transformation.py +++ b/litellm/llms/runwayml/text_to_speech/transformation.py @@ -18,6 +18,7 @@ from litellm.constants import ( RUNWAYML_DEFAULT_API_VERSION, RUNWAYML_POLLING_TIMEOUT, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, TextToSpeechRequestData, @@ -230,10 +231,8 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): """ Get the complete URL for RunwayML TTS request """ - complete_url = api_base or get_secret_str("RUNWAYML_API_BASE") or self.DEFAULT_BASE_URL - - complete_url = complete_url.rstrip("/") - return f"{complete_url}/{self.TTS_ENDPOINT_PATH}" + base_url: Final = api_base or get_secret_str("RUNWAYML_API_BASE") or self.DEFAULT_BASE_URL + return join_api_base(base_url, self.TTS_ENDPOINT_PATH) @staticmethod def _check_timeout(start_time: float, timeout_secs: float) -> None: @@ -311,8 +310,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): start_time: Final = time.time() # Build task status URL - api_base = api_base.rstrip("/") - task_url: Final = f"{api_base}/v1/tasks/{task_id}" + task_url: Final = join_api_base(api_base, f"v1/tasks/{task_id}") verbose_logger.debug("Polling RunwayML TTS task: %s", task_url) @@ -359,8 +357,7 @@ class RunwayMLTextToSpeechConfig(BaseTextToSpeechConfig): start_time: Final = time.time() # Build task status URL - api_base = api_base.rstrip("/") - task_url: Final = f"{api_base}/v1/tasks/{task_id}" + task_url: Final = join_api_base(api_base, f"v1/tasks/{task_id}") verbose_logger.debug("Polling RunwayML TTS task (async): %s", task_url) diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 8b765b1390b..106567f6fba 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -9,7 +9,7 @@ from typing_extensions import ReadOnly, TypedDict import litellm from litellm.constants import RUNWAYML_DEFAULT_API_VERSION -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.custom_httpx.http_handler import ( @@ -274,7 +274,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): files_list: Final[RequestFiles] = [] - return request_data, files_list, f"{api_base}/{endpoint}" + return request_data, files_list, join_api_base(api_base, endpoint) def _select_generation_endpoint(self, request_data: Mapping[str, object]) -> str: if request_data.get("promptVideo") is not None or request_data.get("videoUri") is not None: @@ -413,7 +413,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): encoded_video_id: Final = encode_url_path_segment(original_video_id, field_name="video_id") # Get task status to retrieve video URL - url: Final = f"{api_base}/tasks/{encoded_video_id}" + url: Final = join_api_base(api_base, f"tasks/{encoded_video_id}") return url, dict[str, str]() @@ -571,7 +571,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): encoded_video_id: Final = encode_url_path_segment(original_video_id, field_name="video_id") # Construct the URL for task cancellation - url: Final = f"{api_base}/tasks/{encoded_video_id}/cancel" + url: Final = join_api_base(api_base, f"tasks/{encoded_video_id}/cancel") return url, dict[str, str]() @@ -608,7 +608,7 @@ class RunwayMLVideoConfig(BaseVideoConfig): encoded_video_id: Final = encode_url_path_segment(original_video_id, field_name="video_id") # Construct the full URL for task status retrieval - url: Final = f"{api_base}/tasks/{encoded_video_id}" + url: Final = join_api_base(api_base, f"tasks/{encoded_video_id}") # Empty dict for GET request (no body) return url, dict[str, str]() diff --git a/litellm/llms/s3_vectors/vector_stores/transformation.py b/litellm/llms/s3_vectors/vector_stores/transformation.py index 044315168d2..2b0de3dc9ea 100644 --- a/litellm/llms/s3_vectors/vector_stores/transformation.py +++ b/litellm/llms/s3_vectors/vector_stores/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.vector_store.transformation import ( BaseQueryEmbeddingVectorStoreConfig, VectorStoreEmbeddingExecutor, @@ -131,7 +132,7 @@ class S3VectorsVectorStoreConfig(BaseQueryEmbeddingVectorStoreConfig, BaseAWSLLM litellm_logging_obj: LiteLLMLoggingObj, ) -> tuple[str, dict[str, object]]: litellm_logging_obj.model_call_details["query"] = query_text - return f"{api_base}/QueryVectors", { + return join_api_base(api_base, "QueryVectors"), { "vectorBucketName": bucket_name, "indexName": index_name, "queryVector": {"float32": list(query_vector)}, diff --git a/litellm/llms/sambanova/embedding/transformation.py b/litellm/llms/sambanova/embedding/transformation.py index cf39c0a092f..93cbb13ac47 100644 --- a/litellm/llms/sambanova/embedding/transformation.py +++ b/litellm/llms/sambanova/embedding/transformation.py @@ -10,6 +10,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str @@ -37,11 +38,7 @@ class SambaNovaEmbeddingConfig(BaseEmbeddingConfig): ) -> str: if api_base is None: raise ValueError("api_base is required for SambaNova embeddings") - # Remove trailing slashes and ensure clean base URL - api_base = api_base.rstrip("/") - if not api_base.endswith("/embeddings"): - api_base = f"{api_base}/embeddings" - return api_base + return ensure_api_route(api_base, "embeddings") def validate_environment( self, diff --git a/litellm/llms/sap/chat/transformation.py b/litellm/llms/sap/chat/transformation.py index a82dfe72322..8a7baf18e8f 100755 --- a/litellm/llms/sap/chat/transformation.py +++ b/litellm/llms/sap/chat/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import BaseModel import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse @@ -185,12 +186,12 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): # Keep a short, tight client lifecycle here to avoid fd leaks client: Final = litellm.module_level_client # with httpx.Client(timeout=30) as client: - deployments: Final = client.get(f"{self.base_url}/lm/deployments", headers=self.headers).json() + deployments: Final = client.get(join_api_base(self.base_url, "lm/deployments"), headers=self.headers).json() valid: Final[list[tuple[str, str]]] = [] for dep in deployments.get("resources", []): if dep.get("scenarioId") == "orchestration": cfg = client.get( - f"{self.base_url}/lm/configurations/{dep['configurationId']}", + join_api_base(self.base_url, f"lm/configurations/{dep['configurationId']}"), headers=self.headers, ).json() if cfg.get("executableId") == "orchestration": @@ -263,7 +264,7 @@ class GenAIHubOrchestrationConfig(OpenAIGPTConfig): litellm_params: dict, stream: bool | None = None, ): - api_base_: Final = f"{self.deployment_url}/v2/completion" + api_base_: Final = join_api_base(self.deployment_url, "v2/completion") return api_base_ def _build_prompt_module( diff --git a/litellm/llms/sap/credentials.py b/litellm/llms/sap/credentials.py index b1618f38623..2622d0fa9b6 100644 --- a/litellm/llms/sap/credentials.py +++ b/litellm/llms/sap/credentials.py @@ -16,6 +16,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_httpx_client AUTH_ENDPOINT_SUFFIX: Final = "/oauth/token" @@ -112,7 +113,7 @@ CREDENTIAL_VALUES: Final[list[CredentialsValue]] = [ CredentialsValue( "base_url", ("serviceurls", "AI_API_URL"), - transform_fn=lambda url: url.rstrip("/") + ("" if url.endswith("/v2") else "/v2"), + transform_fn=lambda url: ensure_api_route(url, "v2"), ), CredentialsValue( "cert_url", diff --git a/litellm/llms/sap/embed/transformation.py b/litellm/llms/sap/embed/transformation.py index e91dfb91081..107fef0b355 100644 --- a/litellm/llms/sap/embed/transformation.py +++ b/litellm/llms/sap/embed/transformation.py @@ -8,6 +8,7 @@ from typing import Final, Literal import httpx from pydantic import Field +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.embedding.transformation import ( BaseEmbeddingConfig, LiteLLMLoggingObj, @@ -101,11 +102,11 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig): def deployment_url(self) -> str: with httpx.Client(timeout=30) as client: valid_deployments: Final = [] - deployments: Final = client.get(self.base_url + "/lm/deployments", headers=self.headers).json() + deployments: Final = client.get(join_api_base(self.base_url, "lm/deployments"), headers=self.headers).json() for deployment in deployments.get("resources", []): if deployment["scenarioId"] == "orchestration": config_details = client.get( - self.base_url + f"/lm/configurations/{deployment['configurationId']}", + join_api_base(self.base_url, f"lm/configurations/{deployment['configurationId']}"), headers=self.headers, ).json() if config_details["executableId"] == "orchestration": @@ -144,7 +145,7 @@ class GenAIHubEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - url: Final = self.deployment_url.rstrip("/") + "/v2/embeddings" + url: Final = join_api_base(self.deployment_url, "v2/embeddings") return url def transform_embedding_request( diff --git a/litellm/llms/scaledown/chat/transformation.py b/litellm/llms/scaledown/chat/transformation.py index 44eb4785615..3224bc97148 100644 --- a/litellm/llms/scaledown/chat/transformation.py +++ b/litellm/llms/scaledown/chat/transformation.py @@ -9,6 +9,7 @@ from urllib.parse import unquote import httpx from pydantic import JsonValue, TypeAdapter +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -103,8 +104,8 @@ class ScaleDownChatConfig(BaseConfig): base: Final = _root(api_base or _trusted_root()) operation: Final = _operation(model) if operation in DECISIONS_MODELS: - return f"{base}/v1/scaledown" - return f"{base}{NATIVE_PATHS[operation]}" + return join_api_base(base, "v1/scaledown") + return join_api_base(base, NATIVE_PATHS[operation]) def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseConfig signature operation: Final = _operation(model) diff --git a/litellm/llms/scaleway/audio_transcription/transformation.py b/litellm/llms/scaleway/audio_transcription/transformation.py index 7ebf4475355..8913cc362b3 100644 --- a/litellm/llms/scaleway/audio_transcription/transformation.py +++ b/litellm/llms/scaleway/audio_transcription/transformation.py @@ -9,6 +9,7 @@ from typing import Final import httpx from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -58,8 +59,7 @@ class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - api_base = "https://api.scaleway.ai/v1" if api_base is None else api_base.rstrip("/") - return f"{api_base}/audio/transcriptions" + return join_api_base("https://api.scaleway.ai/v1" if api_base is None else api_base, "audio/transcriptions") def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException: return ScalewayAudioTranscriptionException( diff --git a/litellm/llms/scaleway/rerank/transformation.py b/litellm/llms/scaleway/rerank/transformation.py index 921252d7091..1de5d71f17a 100644 --- a/litellm/llms/scaleway/rerank/transformation.py +++ b/litellm/llms/scaleway/rerank/transformation.py @@ -9,6 +9,7 @@ API reference: https://www.scaleway.com/en/developers/api/generative-apis/#path- from collections.abc import Mapping from typing import Final +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.jina_ai.rerank.transformation import JinaAIRerankConfig from litellm.secret_managers.main import get_secret_str @@ -25,8 +26,8 @@ class ScalewayRerankConfig(JinaAIRerankConfig): model: str, optional_params: Mapping[str, object] | None = None, ) -> str: - base: Final = SCALEWAY_API_BASE if api_base is None else api_base.rstrip("/") - return f"{base}/rerank" + base: Final = SCALEWAY_API_BASE if api_base is None else api_base + return join_api_base(base, "rerank") def validate_environment( self, diff --git a/litellm/llms/searxng/search/transformation.py b/litellm/llms/searxng/search/transformation.py index 265a3db72f5..b1753821cd2 100644 --- a/litellm/llms/searxng/search/transformation.py +++ b/litellm/llms/searxng/search/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -101,12 +102,7 @@ class SearXNGSearchConfig(BaseSearchConfig): "or pass `api_base` parameter. Example: os.environ['SEARXNG_API_BASE'] = 'https://your-searxng-instance.com'" ) - # Append "/search" to the api base if it's not already there - if not api_base.endswith("/search"): - if api_base.endswith("/"): - api_base = f"{api_base}search" - else: - api_base = f"{api_base}/search" + api_base = ensure_api_route(api_base, "search") # Build query parameters from the transformed request body if data and isinstance(data, dict) and "_searxng_params" in data: diff --git a/litellm/llms/serper/search/transformation.py b/litellm/llms/serper/search/transformation.py index 2539610ea59..e82427e4d0c 100644 --- a/litellm/llms/serper/search/transformation.py +++ b/litellm/llms/serper/search/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -84,12 +85,7 @@ class SerperSearchConfig(BaseSearchConfig): Get complete URL for Search endpoint. """ api_base = api_base or get_secret_str("SERPER_API_BASE") or self.SERPER_API_BASE - api_base = api_base.rstrip("/") - - if not api_base.endswith("/search"): - api_base = f"{api_base}/search" - - return api_base + return ensure_api_route(api_base, "search") def transform_search_request( self, diff --git a/litellm/llms/snowflake/chat/transformation.py b/litellm/llms/snowflake/chat/transformation.py index 6a480904a3e..030413030eb 100644 --- a/litellm/llms/snowflake/chat/transformation.py +++ b/litellm/llms/snowflake/chat/transformation.py @@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( select_anthropic_content_block_type_for_file, ) from litellm.litellm_core_utils.prompt_templates.image_handling import async_inline_remote_media +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.chat.handler import ModelResponseIterator as AnthropicStreamParser from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.anthropic.common_utils import normalize_cache_control_in_anthropic_payload @@ -239,8 +240,8 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig): ) -> str: api_base = self._get_api_base(api_base, optional_params) if is_snowflake_claude_model(model): - return f"{api_base}/cortex/v1/messages" - return f"{api_base}/cortex/v1/chat/completions" + return join_api_base(api_base, "cortex/v1/messages") + return join_api_base(api_base, "cortex/v1/chat/completions") def validate_environment( self, diff --git a/litellm/llms/snowflake/embedding/transformation.py b/litellm/llms/snowflake/embedding/transformation.py index ec482337e52..45bec1283a3 100644 --- a/litellm/llms/snowflake/embedding/transformation.py +++ b/litellm/llms/snowflake/embedding/transformation.py @@ -5,6 +5,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.types.llms.openai import AllEmbeddingInputValues @@ -31,7 +32,7 @@ class SnowflakeEmbeddingConfig(SnowflakeBaseConfig, BaseEmbeddingConfig): ) -> str: api_base = self._get_api_base(api_base, optional_params) - return f"{api_base}/cortex/inference:embed" + return join_api_base(api_base, "cortex/inference:embed") def transform_embedding_request( self, diff --git a/litellm/llms/snowflake/utils.py b/litellm/llms/snowflake/utils.py index a8c14d1d7f3..7cc88ab66e1 100644 --- a/litellm/llms/snowflake/utils.py +++ b/litellm/llms/snowflake/utils.py @@ -1,6 +1,7 @@ import re from typing import TYPE_CHECKING, Any, Final +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -53,7 +54,7 @@ class SnowflakeBaseConfig: optional_params[param] = value return optional_params - def _get_api_base(self, api_base, optional_params): + def _get_api_base(self, api_base: str | None, optional_params) -> str: if not api_base: if "account_id" in optional_params: account_id = optional_params.pop("account_id") @@ -65,10 +66,7 @@ class SnowflakeBaseConfig: raise ValueError("Invalid account_id format") api_base = f"https://{account_id}.snowflakecomputing.com/api/v2" - api_base = api_base.rstrip("/") - if not api_base.endswith("/api/v2"): - api_base += "/api/v2" - return api_base + return ensure_api_route(api_base, "api/v2") def validate_environment( self, diff --git a/litellm/llms/soniox/audio_transcription/handler.py b/litellm/llms/soniox/audio_transcription/handler.py index f86bc142898..a48e3e3ed4c 100644 --- a/litellm/llms/soniox/audio_transcription/handler.py +++ b/litellm/llms/soniox/audio_transcription/handler.py @@ -31,6 +31,7 @@ from litellm.litellm_core_utils.audio_utils.utils import ( get_audio_file_name, process_audio_file, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -329,7 +330,7 @@ class SonioxAudioTranscriptionHandler: input=None, api_key=api_key, additional_args={ - "api_base": f"{api_base}/v1/transcriptions", + "api_base": join_api_base(api_base, "v1/transcriptions"), "atranscription": True, "complete_input_dict": SonioxAudioTranscriptionHandler._redact_body_for_logging(body), }, @@ -451,7 +452,7 @@ class SonioxAudioTranscriptionHandler: self._safe_log_pre_call(logging_obj, api_key, base_url, body) create_resp: Final = http_client.post( - url=f"{base_url}/v1/transcriptions", + url=join_api_base(base_url, "v1/transcriptions"), headers=auth_headers, json=body, timeout=timeout, @@ -472,7 +473,7 @@ class SonioxAudioTranscriptionHandler: ) transcript_resp: Final = http_client.get( - url=f"{base_url}/v1/transcriptions/{transcription_id}/transcript", + url=join_api_base(base_url, f"v1/transcriptions/{transcription_id}/transcript"), headers=auth_headers, timeout=timeout, ) @@ -528,7 +529,7 @@ class SonioxAudioTranscriptionHandler: # `Authorization` header is fine; httpx sets multipart Content-Type. upload_headers: Final = {"Authorization": auth_headers["Authorization"]} resp: Final = http_client.post( - url=f"{base_url}/v1/files", + url=join_api_base(base_url, "v1/files"), headers=upload_headers, files=files, timeout=timeout, @@ -550,7 +551,7 @@ class SonioxAudioTranscriptionHandler: ) -> _TranscriptionMeta: for _ in range(max_attempts): resp = http_client.get( - url=f"{base_url}/v1/transcriptions/{transcription_id}", + url=join_api_base(base_url, f"v1/transcriptions/{transcription_id}"), headers=auth_headers, timeout=timeout, ) @@ -593,7 +594,7 @@ class SonioxAudioTranscriptionHandler: if "transcription" in cleanup and transcription_id: try: http_client.delete( - url=f"{base_url}/v1/transcriptions/{transcription_id}", + url=join_api_base(base_url, f"v1/transcriptions/{transcription_id}"), headers=auth_headers, timeout=timeout, ) @@ -605,7 +606,7 @@ class SonioxAudioTranscriptionHandler: if "file" in cleanup and file_id_to_cleanup: try: http_client.delete( - url=f"{base_url}/v1/files/{file_id_to_cleanup}", + url=join_api_base(base_url, f"v1/files/{file_id_to_cleanup}"), headers=auth_headers, timeout=timeout, ) @@ -685,7 +686,7 @@ class SonioxAudioTranscriptionHandler: self._safe_log_pre_call(logging_obj, api_key, base_url, body) create_resp: Final = await http_client.post( - url=f"{base_url}/v1/transcriptions", + url=join_api_base(base_url, "v1/transcriptions"), headers=auth_headers, json=body, timeout=timeout, @@ -706,7 +707,7 @@ class SonioxAudioTranscriptionHandler: ) transcript_resp: Final = await http_client.get( - url=f"{base_url}/v1/transcriptions/{transcription_id}/transcript", + url=join_api_base(base_url, f"v1/transcriptions/{transcription_id}/transcript"), headers=auth_headers, timeout=timeout, ) @@ -761,7 +762,7 @@ class SonioxAudioTranscriptionHandler: } upload_headers: Final = {"Authorization": auth_headers["Authorization"]} resp: Final = await http_client.post( - url=f"{base_url}/v1/files", + url=join_api_base(base_url, "v1/files"), headers=upload_headers, files=files, timeout=timeout, @@ -783,7 +784,7 @@ class SonioxAudioTranscriptionHandler: ) -> _TranscriptionMeta: for _ in range(max_attempts): resp = await http_client.get( - url=f"{base_url}/v1/transcriptions/{transcription_id}", + url=join_api_base(base_url, f"v1/transcriptions/{transcription_id}"), headers=auth_headers, timeout=timeout, ) @@ -826,7 +827,7 @@ class SonioxAudioTranscriptionHandler: if "transcription" in cleanup and transcription_id: try: await http_client.delete( - url=f"{base_url}/v1/transcriptions/{transcription_id}", + url=join_api_base(base_url, f"v1/transcriptions/{transcription_id}"), headers=auth_headers, timeout=timeout, ) @@ -838,7 +839,7 @@ class SonioxAudioTranscriptionHandler: if "file" in cleanup and file_id_to_cleanup: try: await http_client.delete( - url=f"{base_url}/v1/files/{file_id_to_cleanup}", + url=join_api_base(base_url, f"v1/files/{file_id_to_cleanup}"), headers=auth_headers, timeout=timeout, ) diff --git a/litellm/llms/stability/image_generation/transformation.py b/litellm/llms/stability/image_generation/transformation.py index 1856523ce49..a53fd087098 100644 --- a/litellm/llms/stability/image_generation/transformation.py +++ b/litellm/llms/stability/image_generation/transformation.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) @@ -131,11 +132,10 @@ class StabilityImageGenerationConfig(BaseImageGenerationConfig): """ Get the complete URL for the Stability AI API request. """ - base_url: str = api_base or get_secret_str("STABILITY_API_BASE") or self.DEFAULT_BASE_URL - base_url = base_url.rstrip("/") + base_url: Final = api_base or get_secret_str("STABILITY_API_BASE") or self.DEFAULT_BASE_URL endpoint: Final = self._get_model_endpoint(model) - return f"{base_url}{endpoint}" + return join_api_base(base_url, endpoint) def validate_environment( self, diff --git a/litellm/llms/tavily/search/transformation.py b/litellm/llms/tavily/search/transformation.py index d0531e4c041..b46f974e0db 100644 --- a/litellm/llms/tavily/search/transformation.py +++ b/litellm/llms/tavily/search/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -90,11 +91,7 @@ class TavilySearchConfig(BaseSearchConfig): """ Get complete URL for Search endpoint. """ - resolved_api_base: Final = api_base or get_secret_str("TAVILY_API_BASE") or self.TAVILY_API_BASE - clean_api_base: Final = resolved_api_base.rstrip("/") - if clean_api_base.endswith("/search"): - return clean_api_base - return f"{clean_api_base}/search" + return ensure_api_route(api_base or get_secret_str("TAVILY_API_BASE") or self.TAVILY_API_BASE, "search") def transform_search_request( self, diff --git a/litellm/llms/tencent/chat/transformation.py b/litellm/llms/tencent/chat/transformation.py index 7e80b0012df..1f0bcd218d6 100644 --- a/litellm/llms/tencent/chat/transformation.py +++ b/litellm/llms/tencent/chat/transformation.py @@ -9,6 +9,7 @@ from typing import Final, TypedDict from typing_extensions import ReadOnly import litellm +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.secret_managers.main import get_secret_str from litellm.utils import supports_reasoning @@ -137,7 +138,4 @@ class TencentChatConfig(OpenAIGPTConfig): if api_base.endswith("/chat/completions"): return api_base - if not api_base.endswith("/v1"): - api_base = f"{api_base}/v1" - - return f"{api_base}/chat/completions" + return join_api_base(ensure_api_route(api_base, "v1"), "chat/completions") diff --git a/litellm/llms/tencent/messages/transformation.py b/litellm/llms/tencent/messages/transformation.py index c56ecaeb51c..1e52bc37d47 100644 --- a/litellm/llms/tencent/messages/transformation.py +++ b/litellm/llms/tencent/messages/transformation.py @@ -8,6 +8,7 @@ alongside its standard OpenAI-compatible chat completions endpoint. from typing import Any import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.anthropic.pass_through.messages.transformation import ( AnthropicMessagesConfig, ) @@ -82,4 +83,4 @@ class TencentAnthropicMessagesConfig(AnthropicMessagesConfig): elif base_url.endswith("/v1"): base_url = base_url[: -len("/v1")] - return f"{base_url}/v1/messages" + return join_api_base(base_url, "v1/messages") diff --git a/litellm/llms/topaz/image_variations/transformation.py b/litellm/llms/topaz/image_variations/transformation.py index 8d03b73e94e..b3dded36f49 100644 --- a/litellm/llms/topaz/image_variations/transformation.py +++ b/litellm/llms/topaz/image_variations/transformation.py @@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Final from aiohttp import ClientResponse from httpx import Headers, Response +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import ( BaseLLMException, LiteLLMLoggingObj, @@ -39,8 +40,7 @@ class TopazImageVariationConfig(TopazModelInfo, BaseImageVariationConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - clean_api_base: Final = (api_base or "https://api.topazlabs.com").rstrip("/") - return f"{clean_api_base}/image/v1/enhance" + return join_api_base(api_base or "https://api.topazlabs.com", "image/v1/enhance") def map_openai_params( self, diff --git a/litellm/llms/vercel_ai_gateway/chat/transformation.py b/litellm/llms/vercel_ai_gateway/chat/transformation.py index 19c4c2659ca..3fc74b6acc1 100644 --- a/litellm/llms/vercel_ai_gateway/chat/transformation.py +++ b/litellm/llms/vercel_ai_gateway/chat/transformation.py @@ -11,6 +11,7 @@ from typing import Final import httpx import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -92,7 +93,7 @@ class VercelAIGatewayConfig(OpenAIGPTConfig): if api_base is None: api_base = "https://ai-gateway.vercel.sh/v1" - models_url: Final = f"{api_base}/models" + models_url: Final = join_api_base(api_base, "models") response: Final = litellm.module_level_client.get(url=models_url) if response.status_code != 200: diff --git a/litellm/llms/vercel_ai_gateway/embedding/transformation.py b/litellm/llms/vercel_ai_gateway/embedding/transformation.py index 9cfaaab89f0..01d7b7766c0 100644 --- a/litellm/llms/vercel_ai_gateway/embedding/transformation.py +++ b/litellm/llms/vercel_ai_gateway/embedding/transformation.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.secret_managers.main import get_secret_str @@ -77,12 +78,8 @@ class VercelAIGatewayEmbeddingConfig(BaseEmbeddingConfig): """ Get the complete URL for Vercel AI Gateway Embedding API endpoint. """ - if api_base: - api_base = api_base.rstrip("/") - else: - api_base = get_secret_str("VERCEL_AI_GATEWAY_API_BASE") or "https://ai-gateway.vercel.sh/v1" - - return f"{api_base}/embeddings" + base: Final = api_base or get_secret_str("VERCEL_AI_GATEWAY_API_BASE") or "https://ai-gateway.vercel.sh/v1" + return join_api_base(base, "embeddings") def transform_embedding_request( self, diff --git a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py index f4db5eb110c..39d2e9b4c49 100644 --- a/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/gemini_transcribe_transformation.py @@ -10,6 +10,7 @@ from litellm.litellm_core_utils.audio_utils.utils import ( normalize_transcription_language_to_bcp47, process_audio_file, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -129,10 +130,10 @@ class VertexGeminiAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexB project_id: Final = validate_vertex_transcription_project_id( self.safe_get_vertex_ai_project(vertex_params) or self._resolve_project_id_from_credentials(vertex_params) ) - base_url: Final = (api_base or get_vertex_base_url(location)).rstrip("/") + base_url: Final = api_base or get_vertex_base_url(location) bare_model: Final = model.removeprefix("vertex_ai/") model_path: Final = f"projects/{project_id}/locations/{location}/publishers/google/models/{bare_model}" - return f"{base_url}/v1/{model_path}:generateContent" + return join_api_base(base_url, f"v1/{model_path}:generateContent") def _resolve_project_id_from_credentials(self, litellm_params: Mapping[str, object]) -> str: vertex_params: Final = dict(litellm_params) diff --git a/litellm/llms/vertex_ai/audio_transcription/transformation.py b/litellm/llms/vertex_ai/audio_transcription/transformation.py index 5603be81095..4fdf93a6307 100644 --- a/litellm/llms/vertex_ai/audio_transcription/transformation.py +++ b/litellm/llms/vertex_ai/audio_transcription/transformation.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.audio_utils.utils import ( normalize_transcription_language_to_bcp47, process_audio_file, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.audio_transcription.transformation import ( AudioTranscriptionRequestData, BaseAudioTranscriptionConfig, @@ -126,8 +127,8 @@ class VertexAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig, VertexBase) project_id: Final = validate_vertex_transcription_project_id( self.safe_get_vertex_ai_project(litellm_params) or self._resolve_project_id_from_credentials(litellm_params) ) - base_url: Final = (api_base or f"https://{speech_to_text_host(location)}").rstrip("/") - return f"{base_url}/v2/projects/{project_id}/locations/{location}/recognizers/_:recognize" + base_url: Final = api_base or f"https://{speech_to_text_host(location)}" + return join_api_base(base_url, f"v2/projects/{project_id}/locations/{location}/recognizers/_:recognize") def _resolve_project_id_from_credentials(self, litellm_params: dict) -> str: _, project_id = self._ensure_access_token( diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index de5fe39cfc9..7c4238e3498 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -10,6 +10,7 @@ import litellm from litellm.litellm_core_utils.url_utils import ( async_safe_get, encode_url_path_segment, + join_api_base, safe_get, ) from litellm.llms.custom_httpx.http_handler import ( @@ -194,7 +195,7 @@ class VertexAIBatchPrediction(VertexLLM): api_base_path: Final = urlparse(api_base).path.rstrip("/") if api_base_path in ("/v1", "/v1beta1"): return graft_default_vertex_path(api_base=api_base, default_url=default_endpoint_url) - return api_base.rstrip("/") + urlparse(default_endpoint_url).path + return join_api_base(api_base, urlparse(default_endpoint_url).path) def _resolve_fine_tuned_endpoint_model( self, diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index d1e3f10fdd4..e36aba00293 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -37,6 +37,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( extract_file_data, extract_file_metadata, ) +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body @@ -1083,12 +1084,12 @@ class VertexAIFilesConfig(VertexBase, BaseFilesConfig): if object_prefix: object_name = f"{object_prefix}/{object_name}" encoded_object_name: Final = encode_gcs_object_name_for_url(object_name) - endpoint: Final = f"upload/storage/v1/b/{bucket_name}/o?uploadType=media&name={encoded_object_name}" + upload_path: Final = f"upload/storage/v1/b/{bucket_name}/o" api_base = api_base or "https://storage.googleapis.com" if not api_base: raise ValueError("api_base is required") - return f"{api_base}/{endpoint}" + return f"{join_api_base(api_base, upload_path)}?uploadType=media&name={encoded_object_name}" def get_supported_openai_params(self, model: str) -> list[OpenAICreateFileRequestOptionalParams]: return [] diff --git a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py index f916d494aad..085ff715c30 100644 --- a/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py +++ b/litellm/llms/vertex_ai/image_edit/vertex_imagen_transformation.py @@ -12,6 +12,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM @@ -143,12 +144,12 @@ class VertexAIImagenImageEditConfig(BaseImageEditConfig, VertexLLM): if model.startswith("vertex_ai/"): model_name = model.replace("vertex_ai/", "") - if api_base: - base_url = api_base.rstrip("/") - else: - base_url = get_vertex_base_url(vertex_location) + base_url: Final = api_base or get_vertex_base_url(vertex_location) - return f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict" + return join_api_base( + base_url, + f"v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}:predict", + ) def transform_image_edit_request( self, diff --git a/litellm/llms/vertex_ai/interactions/transformation.py b/litellm/llms/vertex_ai/interactions/transformation.py index 36965fe666f..b8e13059917 100644 --- a/litellm/llms/vertex_ai/interactions/transformation.py +++ b/litellm/llms/vertex_ai/interactions/transformation.py @@ -2,7 +2,7 @@ from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import Final -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig from litellm.llms.vertex_ai.common_utils import validate_vertex_location from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -22,9 +22,9 @@ class VertexInteractionsTarget: @property def collection_url(self) -> str: - return ( - f"{self.base_url}/{VERTEX_INTERACTIONS_API_VERSION}" - f"/projects/{self.project_id}/locations/{self.location}/interactions" + return join_api_base( + self.base_url, + f"{VERTEX_INTERACTIONS_API_VERSION}/projects/{self.project_id}/locations/{self.location}/interactions", ) def interaction_url(self, interaction_id: str) -> str: diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index 55f1cf58cbe..13f547b3b92 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -21,6 +21,7 @@ from collections.abc import Awaitable, Callable from typing import Final from litellm import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig from litellm.llms.vertex_ai.audio_transcription.realtime_transformation import ( VertexChirpRealtimeConfig, @@ -68,9 +69,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): """ if api_base: # Allow callers to supply a fully-qualified wss:// base URL. - base = api_base.rstrip("/") - base = base.replace("https://", "wss://").replace("http://", "ws://") - return f"{base}/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent" + base: Final = api_base.replace("https://", "wss://").replace("http://", "ws://") + return join_api_base(base, "ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent") host: Final = get_vertex_base_url(self._location).removeprefix("https://") return f"wss://{host}/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent" diff --git a/litellm/llms/vertex_ai/text_to_speech/transformation.py b/litellm/llms/vertex_ai/text_to_speech/transformation.py index 1441d01e41e..c0082bc5ca6 100644 --- a/litellm/llms/vertex_ai/text_to_speech/transformation.py +++ b/litellm/llms/vertex_ai/text_to_speech/transformation.py @@ -23,7 +23,7 @@ from litellm.litellm_core_utils.audio_utils.utils import ( speech_media_type_from_audio_bytes, ) from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.text_to_speech.transformation import ( BaseTextToSpeechConfig, @@ -1075,13 +1075,14 @@ class VertexAILyriaTextToSpeechConfig(VertexAITextToSpeechConfig): }, ) location: Final = self.safe_get_vertex_ai_location(litellm_params) or self.get_default_vertex_location() - base_url: Final = self.get_api_base(api_base=api_base, vertex_location=location).rstrip("/") + base_url: Final = self.get_api_base(api_base=api_base, vertex_location=location) encoded_project: Final = encode_url_path_segment(project, field_name="project") encoded_location: Final = encode_url_path_segment(location, field_name="location") encoded_model: Final = encode_url_path_segment(base_model, field_name="model") - return ( - f"{base_url}/v1/projects/{encoded_project}/locations/{encoded_location}" - f"/publishers/google/models/{encoded_model}:predict" + return join_api_base( + base_url, + f"v1/projects/{encoded_project}/locations/{encoded_location}" + f"/publishers/google/models/{encoded_model}:predict", ) def transform_text_to_speech_request( diff --git a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py index 5c250fc1a7e..440604688bb 100644 --- a/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any, Final, Protocol import httpx from typing_extensions import ReadOnly, TypedDict +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -288,7 +289,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase): """ Transform create request for Vertex AI RAG Corpus """ - url: Final = f"{api_base}/ragCorpora" # Base URL for creating RAG corpus + url: Final = join_api_base(api_base, "ragCorpora") # Base URL for creating RAG corpus # Add metadata if provided metadata: Final = vector_store_create_optional_params.get("metadata") diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py index 7100c9efde3..4be41e26326 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py @@ -8,6 +8,7 @@ their respective publisher-specific count-tokens endpoints. from typing import Any, Final +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.vertex_ai.common_utils import get_vertex_base_url from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -59,16 +60,14 @@ class VertexAIPartnerModelsTokenCounter(VertexBase): publisher: Final = self._get_publisher_for_model(model) # Use custom api_base if provided, otherwise construct default - if api_base: - base_url = api_base - else: - base_url = get_vertex_base_url(vertex_location) + base_url: Final = api_base or get_vertex_base_url(vertex_location) # Construct the count-tokens endpoint # Format: /v1/projects/{project}/locations/{location}/publishers/{publisher}/models/count-tokens:rawPredict - endpoint: Final = ( - f"{base_url}/v1/projects/{project_id}/locations/{vertex_location}/" - f"publishers/{publisher}/models/count-tokens:rawPredict" + endpoint: Final = join_api_base( + base_url, + f"v1/projects/{project_id}/locations/{vertex_location}/" + f"publishers/{publisher}/models/count-tokens:rawPredict", ) return endpoint diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 77508096e4a..6cff9729189 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -14,6 +14,7 @@ from urllib.parse import urlparse import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.secret_managers.main import get_secret_str from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES, VertexPartnerProvider @@ -295,23 +296,22 @@ class VertexBase: if api_base is None: api_base = get_vertex_base_url(vertex_location) + location_path: Final = f"projects/{vertex_project}/locations/{vertex_location}" + raw_predict_method: Final = "streamRawPredict" if stream else "rawPredict" if partner == VertexPartnerProvider.llama: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/endpoints/openapi/chat/completions" + return join_api_base(api_base, f"v1/{location_path}/endpoints/openapi/chat/completions") elif partner == VertexPartnerProvider.mistralai: - if stream: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:streamRawPredict" - else: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:rawPredict" + return join_api_base( + api_base, f"v1/{location_path}/publishers/mistralai/models/{model}:{raw_predict_method}" + ) elif partner == VertexPartnerProvider.ai21: - if stream: - return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:streamRawPredict" - else: - return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:rawPredict" + return join_api_base( + api_base, f"v1beta1/{location_path}/publishers/ai21/models/{model}:{raw_predict_method}" + ) elif partner == VertexPartnerProvider.claude: - if stream: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:streamRawPredict" - else: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:rawPredict" + return join_api_base( + api_base, f"v1/{location_path}/publishers/anthropic/models/{model}:{raw_predict_method}" + ) def get_complete_vertex_url( self, @@ -658,7 +658,7 @@ class VertexBase: # For Gemini (Google AI Studio), construct the full path like other providers if model is None: raise ValueError("Model parameter is required for Gemini custom API base URLs") - url = f"{api_base}/models/{model}:{endpoint}" + url = join_api_base(api_base, f"models/{model}:{endpoint}") if gemini_api_key is None: raise ValueError( "Missing Gemini API key. Set the GEMINI_API_KEY or GOOGLE_API_KEY environment variable." @@ -678,20 +678,17 @@ class VertexBase: model_for_url: Final = get_vertex_base_model_name(model=model) # Format: {api_base}/v1/projects/{project}/locations/{location}/endpoints/{model}:{endpoint} version: Final = vertex_api_version or "v1" - url = "{}/{}/projects/{}/locations/{}/endpoints/{}:{}".format( - api_base.rstrip("/"), - version, - vertex_project, - vertex_location, - model_for_url, - endpoint, + url = join_api_base( + api_base, + f"{version}/projects/{vertex_project}/locations/{vertex_location}" + f"/endpoints/{model_for_url}:{endpoint}", ) elif urlparse(api_base).path in ("", "/"): - url = api_base.rstrip("/") + urlparse(url).path + url = join_api_base(api_base, urlparse(url).path) elif urlparse(api_base).path.rstrip("/") in ("/v1", "/v1beta1") and "/projects/" in urlparse(url).path: url = graft_default_vertex_path(api_base=api_base, default_url=url) else: - url = f"{api_base}:{endpoint}" + url = f"{api_base.rstrip('/')}:{endpoint}" if stream is True: parsed_stream_url: Final = urlparse(url) stream_query: Final = f"{parsed_stream_url.query}&alt=sse" if parsed_stream_url.query else "alt=sse" diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index de368cea5ac..080899d08b0 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -18,6 +18,7 @@ from typing_extensions import ReadOnly import litellm from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS from litellm.images.utils import ImageEditRequestUtils +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.videos.transformation import BaseVideoConfig from litellm.llms.vertex_ai.common_utils import ( convert_vertex_datetime_to_openai_datetime, @@ -325,13 +326,10 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): model_name: Final = model.replace("vertex_ai/", "") # Construct the URL - if api_base: - base_url = api_base.rstrip("/") - else: - base_url = get_vertex_base_url(vertex_location) + base_url: Final = api_base or get_vertex_base_url(vertex_location) - url: Final = ( - f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}" + url: Final = join_api_base( + base_url, f"v1/projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{model_name}" ) return url @@ -480,7 +478,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): # Construct the full URL including model ID # URL format: https://LOCATION-aiplatform.googleapis.com/v1/projects/PROJECT/locations/LOCATION/publishers/google/models/MODEL:fetchPredictOperation # Strip trailing slashes from api_base and append model - url: Final = f"{api_base.rstrip('/')}/{model}:fetchPredictOperation" + url: Final = join_api_base(api_base, f"{model}:fetchPredictOperation") # Request body contains the operation name params: Final = {"operationName": operation_name} @@ -778,7 +776,7 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): if vertex_params: request_data["parameters"] = vertex_params - edit_url: Final = f"{api_base.rstrip('/')}/{model}:predictLongRunning" + edit_url: Final = join_api_base(api_base, f"{model}:predictLongRunning") return edit_url, request_data, None def transform_video_edit_response( diff --git a/litellm/llms/volcengine/embedding/transformation.py b/litellm/llms/volcengine/embedding/transformation.py index b298c9996f7..cc1e7d59caa 100644 --- a/litellm/llms/volcengine/embedding/transformation.py +++ b/litellm/llms/volcengine/embedding/transformation.py @@ -10,6 +10,7 @@ from typing import Final import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues @@ -76,12 +77,8 @@ class VolcEngineEmbeddingConfig(BaseEmbeddingConfig): Returns: Complete URL for the embedding API endpoint """ - base_url: Final = get_volcengine_base_url(api_base) - # Construct the complete URL with /embeddings endpoint - if base_url.endswith("/api/v3"): - return f"{base_url}/embeddings" - else: - return f"{base_url}/api/v3/embeddings" + versioned_base_url: Final = ensure_api_route(get_volcengine_base_url(api_base), "api/v3") + return join_api_base(versioned_base_url, "embeddings") def map_openai_params( self, diff --git a/litellm/llms/volcengine/responses/transformation.py b/litellm/llms/volcengine/responses/transformation.py index 6e0aa88b6b6..74f6182bd89 100644 --- a/litellm/llms/volcengine/responses/transformation.py +++ b/litellm/llms/volcengine/responses/transformation.py @@ -11,7 +11,7 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import ( safe_convert_created_field, ) -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -137,8 +137,8 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): if base_url.endswith("/responses"): return base_url if base_url.endswith("/api/v3"): - return f"{base_url}/responses" - return f"{base_url}/api/v3/responses" + return join_api_base(base_url, "responses") + return join_api_base(base_url, "api/v3/responses") def map_openai_params( self, @@ -345,7 +345,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): order: Literal["asc", "desc"] = "desc", ) -> tuple[str, dict]: encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") - url: Final = f"{api_base}/{encoded_response_id}/input_items" + url: Final = join_api_base(api_base, f"{encoded_response_id}/input_items") params: Final[dict[str, str | int]] = {} if after is not None: params["after"] = after @@ -380,7 +380,7 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig): headers: dict, ) -> tuple[str, dict]: encoded_response_id: Final = encode_url_path_segment(response_id, field_name="response_id") - url: Final = f"{api_base}/{encoded_response_id}/cancel" + url: Final = join_api_base(api_base, f"{encoded_response_id}/cancel") data: Final[dict] = {} return url, data diff --git a/litellm/llms/voyage/embedding/transformation.py b/litellm/llms/voyage/embedding/transformation.py index 7e8425bb3d0..1ba66e5ce65 100644 --- a/litellm/llms/voyage/embedding/transformation.py +++ b/litellm/llms/voyage/embedding/transformation.py @@ -5,6 +5,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.voyage.common_utils import get_default_base_url, get_voyage_api_key @@ -50,12 +51,7 @@ class VoyageEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - if api_base: - clean_api_base: Final = api_base.rstrip("/") - if not clean_api_base.endswith("/embeddings"): - return f"{clean_api_base}/embeddings" - return clean_api_base - return f"{get_default_base_url(api_key)}/embeddings" + return ensure_api_route(api_base or get_default_base_url(api_key), "embeddings") def get_supported_openai_params(self, model: str) -> list: return [ diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 8e3802ccbc9..62c03139fbe 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -10,6 +10,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.voyage.common_utils import get_default_base_url, get_voyage_api_key @@ -58,12 +59,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - if api_base: - clean_api_base: Final = api_base.rstrip("/") - if not clean_api_base.endswith("/contextualizedembeddings"): - return f"{clean_api_base}/contextualizedembeddings" - return clean_api_base - return f"{get_default_base_url(api_key)}/contextualizedembeddings" + return ensure_api_route(api_base or get_default_base_url(api_key), "contextualizedembeddings") def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class signature return ["encoding_format", "dimensions"] diff --git a/litellm/llms/voyage/embedding/transformation_multimodal.py b/litellm/llms/voyage/embedding/transformation_multimodal.py index 29fdb9ecfe1..c04a2e1f26b 100644 --- a/litellm/llms/voyage/embedding/transformation_multimodal.py +++ b/litellm/llms/voyage/embedding/transformation_multimodal.py @@ -13,6 +13,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.voyage.common_utils import get_default_base_url, get_voyage_api_key @@ -59,12 +60,7 @@ class VoyageMultimodalEmbeddingConfig(BaseEmbeddingConfig): litellm_params: dict, stream: bool | None = None, ) -> str: - if api_base: - clean_api_base: Final = api_base.rstrip("/") - if not clean_api_base.endswith("/multimodalembeddings"): - return f"{clean_api_base}/multimodalembeddings" - return clean_api_base - return f"{get_default_base_url(api_key)}/multimodalembeddings" + return ensure_api_route(api_base or get_default_base_url(api_key), "multimodalembeddings") def get_supported_openai_params(self, model: str) -> list: return ["dimensions"] diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index 121c18e82ae..7fe643870d5 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -11,6 +11,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from litellm._uuid import uuid +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.llms.voyage.common_utils import get_default_base_url, get_voyage_api_key @@ -80,9 +81,9 @@ class VoyageRerankConfig(BaseRerankConfig): api_base = api_base.rstrip("/") if not api_base.endswith("/v1/rerank"): if api_base.endswith("/v1"): - api_base = f"{api_base}/rerank" + api_base = join_api_base(api_base, "rerank") else: - api_base = f"{api_base}/v1/rerank" + api_base = join_api_base(api_base, "v1/rerank") return api_base def transform_rerank_request( diff --git a/litellm/llms/watsonx/audio_transcription/transformation.py b/litellm/llms/watsonx/audio_transcription/transformation.py index 77ef03b8e10..e5da7dd412f 100644 --- a/litellm/llms/watsonx/audio_transcription/transformation.py +++ b/litellm/llms/watsonx/audio_transcription/transformation.py @@ -12,6 +12,7 @@ from pydantic import ConfigDict, TypeAdapter import litellm from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.types.llms.openai import ( AllMessageValues, OpenAIAudioTranscriptionOptionalParams, @@ -150,10 +151,9 @@ class IBMWatsonXAudioTranscriptionConfig(IBMWatsonXMixin, OpenAIWhisperAudioTran """ # Get base URL url = self._get_base_url(api_base=api_base) - url = url.rstrip("/") # Add the audio transcription endpoint - url = f"{url}/ml/v1/audio/transcriptions" + url = join_api_base(url, "ml/v1/audio/transcriptions") # Add version parameter (only version in query string, not project_id) api_version: Final = optional_params.get("api_version", None) or litellm.WATSONX_DEFAULT_API_VERSION diff --git a/litellm/llms/watsonx/chat/transformation.py b/litellm/llms/watsonx/chat/transformation.py index 4ff2212711d..071ff7108b9 100644 --- a/litellm/llms/watsonx/chat/transformation.py +++ b/litellm/llms/watsonx/chat/transformation.py @@ -7,6 +7,7 @@ Docs: https://cloud.ibm.com/apidocs/watsonx-ai#text-chat from typing import Final from litellm import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.secret_managers.main import get_secret_str from litellm.types.llms.watsonx import ( WatsonXAIEndpoint, @@ -96,7 +97,7 @@ class IBMWatsonXChatConfig(IBMWatsonXMixin, OpenAIGPTConfig): endpoint = endpoint.format(deployment_id=deployment_id) else: endpoint = WatsonXAIEndpoint.CHAT_STREAM.value if stream else WatsonXAIEndpoint.CHAT.value - url = url.rstrip("/") + endpoint + url = join_api_base(url, endpoint) ## add api version url = self._add_api_version_to_url(url=url, api_version=optional_params.pop("api_version", None)) diff --git a/litellm/llms/watsonx/completion/transformation.py b/litellm/llms/watsonx/completion/transformation.py index 5c0416bd6ca..9ea062ea2f8 100644 --- a/litellm/llms/watsonx/completion/transformation.py +++ b/litellm/llms/watsonx/completion/transformation.py @@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Final import httpx +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock from litellm.types.llms.watsonx import WatsonXAIEndpoint @@ -342,7 +343,7 @@ class IBMWatsonXAIConfig(IBMWatsonXMixin, BaseConfig): endpoint = endpoint.format(deployment_id=deployment_id) else: endpoint = WatsonXAIEndpoint.TEXT_GENERATION_STREAM if stream else WatsonXAIEndpoint.TEXT_GENERATION - url = url.rstrip("/") + endpoint + url = join_api_base(url, endpoint) ## add api version url = self._add_api_version_to_url(url=url, api_version=optional_params.pop("api_version", None)) diff --git a/litellm/llms/watsonx/embed/transformation.py b/litellm/llms/watsonx/embed/transformation.py index 1b870c7f639..6ec3b50bbe9 100644 --- a/litellm/llms/watsonx/embed/transformation.py +++ b/litellm/llms/watsonx/embed/transformation.py @@ -9,6 +9,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter from typing_extensions import ReadOnly, TypedDict +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.embedding.transformation import ( BaseEmbeddingConfig, LiteLLMLoggingObj, @@ -92,7 +93,7 @@ class IBMWatsonXEmbeddingConfig(IBMWatsonXMixin, BaseEmbeddingConfig): if model.startswith("deployment/"): deployment_id: Final = "/".join(model.split("/")[1:]) endpoint = endpoint.format(deployment_id=deployment_id) - url = url.rstrip("/") + endpoint + url = join_api_base(url, endpoint) ## add api version url = self._add_api_version_to_url(url=url, api_version=optional_params.pop("api_version", None)) diff --git a/litellm/llms/watsonx/rerank/transformation.py b/litellm/llms/watsonx/rerank/transformation.py index 5f337603763..591ddda3cbd 100644 --- a/litellm/llms/watsonx/rerank/transformation.py +++ b/litellm/llms/watsonx/rerank/transformation.py @@ -11,6 +11,7 @@ from typing import Final, cast import httpx from pydantic import ConfigDict, TypeAdapter +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig from litellm.secret_managers.main import get_secret_str @@ -45,7 +46,7 @@ class IBMWatsonXRerankConfig(IBMWatsonXMixin, BaseRerankConfig): base_url: Final = self._get_base_url(api_base=api_base) endpoint: Final = WatsonXAIEndpoint.RERANK.value - url: Final = base_url.rstrip("/") + endpoint + url: Final = join_api_base(base_url, endpoint) params: Final = optional_params or {} diff --git a/litellm/llms/xai/audio_transcription/transformation.py b/litellm/llms/xai/audio_transcription/transformation.py index e34034b5ae9..e29c3d9334a 100644 --- a/litellm/llms/xai/audio_transcription/transformation.py +++ b/litellm/llms/xai/audio_transcription/transformation.py @@ -10,6 +10,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.audio_utils.utils import process_audio_file +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ( @@ -185,7 +186,7 @@ class XAIAudioTranscriptionConfig(BaseAudioTranscriptionConfig): ) -> str: base: Final = (XAIModelInfo.get_api_base(api_base) or "").rstrip("/") normalized: Final = base.removesuffix("/v1") - return f"{normalized}/v1/stt" + return join_api_base(normalized, "v1/stt") def validate_environment( self, diff --git a/litellm/llms/xai/batches/transformation.py b/litellm/llms/xai/batches/transformation.py index fdd9939526e..1e803057e3d 100644 --- a/litellm/llms/xai/batches/transformation.py +++ b/litellm/llms/xai/batches/transformation.py @@ -20,7 +20,7 @@ from pydantic import ConfigDict from typing_extensions import NotRequired, ReadOnly, TypedDict from litellm.constants import XAI_API_BASE -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.xai.common_utils import XAIModelInfo from litellm.secret_managers.main import get_secret_str @@ -76,10 +76,11 @@ def get_xai_auth_headers( def xai_batches_url(api_base: str | None, batch_id: str | None = None, suffix: str = "") -> str: - base: Final = f"{get_xai_api_base(api_base)}/v1/batches" + root: Final = get_xai_api_base(api_base) if batch_id is None: - return base - return f"{base}/{encode_url_path_segment(batch_id, field_name='batch_id')}{suffix}" + return join_api_base(root, "v1/batches") + encoded_batch_id: Final = encode_url_path_segment(batch_id, field_name="batch_id") + return join_api_base(root, f"v1/batches/{encoded_batch_id}{suffix}") def is_xai_batch_results_id(file_id: str) -> bool: diff --git a/litellm/llms/xai/common_utils.py b/litellm/llms/xai/common_utils.py index 4843caeac59..06e3f32a767 100644 --- a/litellm/llms/xai/common_utils.py +++ b/litellm/llms/xai/common_utils.py @@ -5,6 +5,7 @@ import httpx from pydantic import ConfigDict, TypeAdapter import litellm +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.base_utils import BaseLLMModelInfo from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues @@ -91,7 +92,7 @@ class XAIModelInfo(BaseLLMModelInfo): "XAI API base or key is not set. Set XAI_API_BASE and provide an xAI API key via api_key, litellm.xai_key, or XAI_API_KEY." ) response: Final = litellm.module_level_client.get( - url=f"{api_base}/v1/models", + url=join_api_base(api_base, "v1/models"), headers={"Authorization": f"Bearer {api_key}"}, ) diff --git a/litellm/llms/xai/files/transformation.py b/litellm/llms/xai/files/transformation.py index 33baca79005..88f2fc51983 100644 --- a/litellm/llms/xai/files/transformation.py +++ b/litellm/llms/xai/files/transformation.py @@ -14,7 +14,7 @@ from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data -from litellm.litellm_core_utils.url_utils import encode_url_path_segment +from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj from litellm.types.llms.base import LiteLLMBaseModel @@ -101,11 +101,11 @@ class XAIFilesConfig(BaseFilesConfig): litellm_params: Mapping[str, object], stream: bool | None = None, ) -> str: - return f"{get_xai_api_base(api_base)}/v1/files" + return join_api_base(get_xai_api_base(api_base), "v1/files") def _file_url(self, file_id: str, litellm_params: Mapping[str, object], suffix: str = "") -> str: encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id") - return f"{_api_base_from(litellm_params)}/v1/files/{encoded_file_id}{suffix}" + return join_api_base(_api_base_from(litellm_params), f"v1/files/{encoded_file_id}{suffix}") def get_error_class( self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers @@ -204,7 +204,7 @@ class XAIFilesConfig(BaseFilesConfig): optional_params: Mapping[str, object], litellm_params: Mapping[str, object], ) -> tuple[str, dict[str, str]]: # mutable-ok: BaseFilesConfig signature - return f"{_api_base_from(litellm_params)}/v1/files", _NO_QUERY_PARAMS + return join_api_base(_api_base_from(litellm_params), "v1/files"), _NO_QUERY_PARAMS def transform_list_files_next_request( self, @@ -215,7 +215,7 @@ class XAIFilesConfig(BaseFilesConfig): page: Final = XAIFileList.model_validate(raw_response.json()) if not page.pagination_token or not page.data: return None - return f"{_api_base_from(litellm_params)}/v1/files", {"pagination_token": page.pagination_token} + return join_api_base(_api_base_from(litellm_params), "v1/files"), {"pagination_token": page.pagination_token} def transform_list_files_response( self, diff --git a/litellm/llms/xai/responses/transformation.py b/litellm/llms/xai/responses/transformation.py index 2f638da49c8..b6ea516fb79 100644 --- a/litellm/llms/xai/responses/transformation.py +++ b/litellm/llms/xai/responses/transformation.py @@ -9,6 +9,7 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import XAI_API_BASE from litellm.exceptions import AuthenticationError +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.llms.xai.common_utils import XAIModelInfo, xai_reported_cost_in_usd from litellm.secret_managers.main import get_secret_str @@ -249,10 +250,7 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): else: api_base = api_base or litellm.api_base or get_secret_str("XAI_API_BASE") or XAI_API_BASE - # Remove trailing slashes - api_base = api_base.rstrip("/") - - return f"{api_base}/responses" + return join_api_base(api_base, "responses") def transform_response_api_response( self, diff --git a/litellm/llms/you_com/search/transformation.py b/litellm/llms/you_com/search/transformation.py index 8b2df949a93..9228deda26b 100644 --- a/litellm/llms/you_com/search/transformation.py +++ b/litellm/llms/you_com/search/transformation.py @@ -10,6 +10,7 @@ from typing import Final, TypedDict import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.base_llm.search.transformation import ( BaseSearchConfig, SearchResponse, @@ -109,7 +110,7 @@ class YouComSearchConfig(BaseSearchConfig): api_base = api_base.rstrip("/") if not api_base.endswith("/v1/search") and not api_base.endswith("/v1/agents/search"): - api_base = f"{api_base}/v1/search" + api_base = join_api_base(api_base, "v1/search") return api_base diff --git a/litellm/main.py b/litellm/main.py index 079b0bcbd6c..27700cf9def 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -112,6 +112,7 @@ from litellm.litellm_core_utils.request_timeout_resolver import ( get_configured_request_timeout, ) from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer +from litellm.litellm_core_utils.url_utils import ensure_api_route from litellm.llms.base_llm import BaseConfig, BaseImageGenerationConfig from litellm.llms.base_llm.base_model_iterator import ( convert_model_response_to_streaming, @@ -230,6 +231,7 @@ from .llms.azure.azure import AzureChatCompletion, check_dynamic_azure_params from .llms.azure.chat.o_series_handler import AzureOpenAIO1ChatCompletion from .llms.azure.completion.handler import AzureTextCompletion from .llms.azure_ai.anthropic.handler import AzureAnthropicChatCompletion +from .llms.azure_ai.anthropic.transformation import azure_anthropic_messages_url from .llms.azure_ai.embed import AzureAIEmbedding from .llms.bedrock.chat import BedrockConverseLLM from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig @@ -1666,16 +1668,8 @@ def _complete_azure_ai(ctx: CompletionDispatchContext) -> _CompletionDispatchRes ) api_key = AzureFoundryModelInfo.get_api_key(api_key) - # Ensure the URL ends with /v1/messages for Anthropic if api_base: - api_base = api_base.rstrip("/") - if not api_base.endswith("/v1/messages"): - if "/anthropic" in api_base: - parts: Final = api_base.split("/anthropic", 1) - api_base = parts[0] + "/anthropic" - else: - api_base = api_base + "/anthropic" - api_base = api_base + "/v1/messages" + api_base = azure_anthropic_messages_url(api_base) response = azure_anthropic_chat_completions.completion( model=model, @@ -2895,8 +2889,8 @@ def _complete_anthropic_text( # Check if we should disable automatic URL suffix appending disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if api_base is not None and not disable_url_suffix and not api_base.endswith("/v1/complete"): - api_base += "/v1/complete" + if api_base is not None and not disable_url_suffix: + api_base = ensure_api_route(api_base, "v1/complete") elif disable_url_suffix: verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix") @@ -2951,8 +2945,8 @@ def _complete_anthropic(ctx: CompletionDispatchContext) -> _CompletionDispatchRe # Check if we should disable automatic URL suffix appending disable_url_suffix: Final = get_secret_bool("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX") - if api_base is not None and not disable_url_suffix and not api_base.endswith("/v1/messages"): - api_base += "/v1/messages" + if api_base is not None and not disable_url_suffix: + api_base = ensure_api_route(api_base, "v1/messages") elif disable_url_suffix: verbose_logger.debug("LITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix") @@ -4090,9 +4084,7 @@ def _complete_text_completion_inception( api_base = passed_api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1" # FIM is served at `/v1/fim/completions`; the OpenAI client appends # `/completions`, so point it at the `/v1/fim` base. - api_base = api_base.rstrip("/") - if not api_base.endswith("/fim"): - api_base += "/fim" + api_base = ensure_api_route(api_base, "fim") # Don't forward the server-managed Inception key to a caller-supplied # api_base; only resolve it for the default/server base, or when the diff --git a/litellm/rag/ingestion/gemini_ingestion.py b/litellm/rag/ingestion/gemini_ingestion.py index 4ee16685e03..6320e477bf2 100644 --- a/litellm/rag/ingestion/gemini_ingestion.py +++ b/litellm/rag/ingestion/gemini_ingestion.py @@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Final, cast from pydantic import ConfigDict from litellm._logging import verbose_logger +from litellm.litellm_core_utils.url_utils import join_api_base from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, header_value, @@ -107,7 +108,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): raise ValueError("GEMINI_API_BASE is required") api_version: Final = "v1beta" - base_url: Final = f"{api_base}/{api_version}" + base_url: Final = join_api_base(api_base, api_version) # Create File Search store if not provided if not vector_store_id: @@ -148,7 +149,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): Returns: Store name (format: fileSearchStores/xxxxxxx) """ - url: Final = f"{base_url}/fileSearchStores" + url: Final = join_api_base(base_url, "fileSearchStores") request_body: Final = {"displayName": display_name} @@ -236,7 +237,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): # base_url is like: https://generativelanguage.googleapis.com/v1beta # We need: https://generativelanguage.googleapis.com/upload/v1beta/{store_id}:uploadToFileSearchStore api_base: Final = base_url.replace("/v1beta", "") # Get base without version - url: Final = f"{api_base}/upload/v1beta/{vector_store_id}:uploadToFileSearchStore" + url: Final = join_api_base(api_base, f"upload/v1beta/{vector_store_id}:uploadToFileSearchStore") # Build request body with chunking config and metadata if provided request_body: Final[dict[str, object]] = {"displayName": filename} diff --git a/tests/integration/providers/_api_base_url_join_support.py b/tests/integration/providers/_api_base_url_join_support.py new file mode 100644 index 00000000000..44bd6ec4dca --- /dev/null +++ b/tests/integration/providers/_api_base_url_join_support.py @@ -0,0 +1,163 @@ +import json +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import Final +from urllib.parse import urlsplit + +from integration._support.anthropic_sse import message_stream +from integration._support.openai_wire import chat_reply +from integration._support.upstream import aws_event_stream_frame +from integration._support.wire import Reply, Request +from integration.providers._api_base_trailing_slash_support import VECTOR, marker_of, reply_text, request_json +from pydantic import JsonValue + +COUNTED_TOKENS: Final = 7 +RELEVANCE: Final = 0.75 +_USAGE: Final = {"input_tokens": 3, "output_tokens": 2} + + +def _json(payload: Mapping[str, JsonValue]) -> Reply: + return Reply(body=json.dumps(payload).encode()) + + +def _path(request: Request) -> str: + return urlsplit(request.target).path + + +def _wants_stream(request: Request) -> bool: + return request_json(request).get("stream") is True + + +def anthropic_message(marker: str) -> dict[str, JsonValue]: + return { + "id": f"msg_{marker}", + "type": "message", + "role": "assistant", + "model": "scripted", + "content": [{"type": "text", "text": reply_text(marker)}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": dict(_USAGE), + } + + +def _anthropic(request: Request) -> Reply: + if _path(request).endswith("/count_tokens"): + return _json({"input_tokens": COUNTED_TOKENS}) + marker: Final = marker_of(request) + if _wants_stream(request): + return Reply( + content_type="text/event-stream", chunks=message_stream(f"msg_{marker}", "scripted", reply_text(marker)) + ) + return _json(anthropic_message(marker)) + + +def _gemini_candidate(text: str, finished: bool) -> dict[str, JsonValue]: + finish: Final[dict[str, JsonValue]] = {"finishReason": "STOP"} if finished else {} + return {"content": {"role": "model", "parts": [{"text": text}]}, "index": 0, **finish} + + +def _gemini_frame(text: str, finished: bool) -> bytes: + usage: Final = {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5} + frame: Final = {"candidates": [_gemini_candidate(text, finished)], "usageMetadata": usage} + return b"data: " + json.dumps(frame).encode() + b"\r\n\r\n" + + +def _gemini(request: Request) -> Reply: + path: Final = _path(request) + marker: Final = marker_of(request) + if path.endswith(":batchEmbedContents"): + return _json({"embeddings": [{"values": list(VECTOR)}]}) + if path.endswith(":streamGenerateContent"): + return Reply( + content_type="text/event-stream", + chunks=(_gemini_frame("reply ", False), _gemini_frame(marker, True)), + ) + return _json( + { + "candidates": [_gemini_candidate(reply_text(marker), True)], + "usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}, + } + ) + + +def _embedding_list(model: str) -> Reply: + return _json( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": list(VECTOR)}], + "model": model, + "usage": {"prompt_tokens": 2, "total_tokens": 2}, + } + ) + + +def _openai_like(model: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + path: Final = _path(request) + marker: Final = marker_of(request) + if path.endswith("/embeddings"): + return _embedding_list(model) + if path.endswith("/audio/transcriptions"): + return _json({"text": reply_text(marker), "model": model}) + return chat_reply(f"chatcmpl-{marker}", model, reply_text(marker), stream=_wants_stream(request)) + + return respond + + +def _converse_event(kind: str, payload: Mapping[str, JsonValue]) -> bytes: + headers: Final = {":event-type": kind, ":content-type": "application/json", ":message-type": "event"} + return aws_event_stream_frame(headers, json.dumps(payload, separators=(",", ":")).encode()) + + +def _bedrock_converse(request: Request) -> Reply: + marker: Final = marker_of(request) + usage: Final[dict[str, JsonValue]] = {"inputTokens": 3, "outputTokens": 2, "totalTokens": 5} + if _path(request).endswith("/converse-stream"): + events: Final[tuple[tuple[str, Mapping[str, JsonValue]], ...]] = ( + ("messageStart", {"role": "assistant"}), + ("contentBlockDelta", {"delta": {"text": "reply "}, "contentBlockIndex": 0}), + ("contentBlockDelta", {"delta": {"text": marker}, "contentBlockIndex": 0}), + ("contentBlockStop", {"contentBlockIndex": 0}), + ("messageStop", {"stopReason": "end_turn"}), + ("metadata", {"usage": usage, "metrics": {"latencyMs": 1}}), + ) + return Reply( + content_type="application/vnd.amazon.eventstream", + chunks=tuple(_converse_event(kind, payload) for kind, payload in events), + ) + return _json( + { + "output": {"message": {"role": "assistant", "content": [{"text": reply_text(marker)}]}}, + "stopReason": "end_turn", + "usage": usage, + "metrics": {"latencyMs": 1}, + } + ) + + +def _cohere_rerank(request: Request) -> Reply: + return _json( + { + "id": f"rerank-{marker_of(request)}", + "results": [{"index": 0, "relevance_score": RELEVANCE}], + "meta": {"billed_units": {"search_units": 1}}, + } + ) + + +_BY_PREFIX: Final[Mapping[str, Callable[[Request], Reply]]] = MappingProxyType( + { + "an": _anthropic, + "bi": lambda request: _json(anthropic_message(marker_of(request))), + "bc": _bedrock_converse, + "gm": _gemini, + "mi": _openai_like("mistral-small-latest"), + "db": _openai_like("databricks-meta-llama-3-3-70b-instruct"), + "co": _cohere_rerank, + } +) + + +def join_peer(request: Request) -> Reply: + return _BY_PREFIX[request.target.split("/")[1]](request) diff --git a/tests/integration/providers/test_api_base_url_join_chaos.py b/tests/integration/providers/test_api_base_url_join_chaos.py new file mode 100644 index 00000000000..8ebcd165f4d --- /dev/null +++ b/tests/integration/providers/test_api_base_url_join_chaos.py @@ -0,0 +1,176 @@ +import asyncio +from collections.abc import Awaitable, Callable +from itertools import chain, repeat +from typing import Final, TypeAlias + +import anthropic +import httpx +import openai +from integration._support.client import Gateway, Scenario, eventually +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from integration.providers._api_base_trailing_slash_support import ( + KEY, + VECTOR, + assert_control_answers, + assert_one_request, + body_of, + marker_of, + new_marker, + of_marker, + reply_text, +) +from integration.providers._api_base_url_join_support import join_peer +from pydantic import JsonValue + +_PER_KIND: Final = 3 +_HAIKU: Final = "anthropic.claude-haiku-5-5" +_GENERATE: Final = "/gm/v1beta/models/gemini-3.8-flash:generateContent" +_STREAM_GENERATE: Final = "/gm/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse" + +Answer: TypeAlias = tuple[str | None, str | None] +Call: TypeAlias = Callable[[str], Awaitable[Answer]] + + +def _deploy(scenario: Scenario, model: str, api_base: str) -> str: + if model.startswith("bedrock/"): + return scenario.model(model=model, api_base=api_base, api_key=KEY, aws_region_name="us-east-1") + return scenario.model(model=model, api_base=api_base, api_key=KEY) + + +def _spend_api_bases(identities: tuple[str, ...]) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + "SELECT request_id, api_base FROM \"LiteLLM_SpendLogs\" WHERE request_id = ANY(string_to_array(%s, ','))", + (",".join(identities),), + ), + lambda found: len(found) == len(identities), + seconds=70, + ) + return {str(row["request_id"]): row["api_base"] for row in rows} + + +def test_mixed_burst_on_slash_bases_lands_each_call_once_on_its_single_slash_url(gateway: Gateway) -> None: + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + claude: Final = _deploy(scenario, "anthropic/claude-haiku-5-5", f"{wire.url}/an/") + gemini: Final = _deploy(scenario, "gemini/gemini-3.8-flash", f"{wire.url}/gm/v1beta/") + mistral: Final = _deploy(scenario, "mistral/mistral-small-latest", f"{wire.url}/mi/v1/") + converse: Final = _deploy(scenario, f"bedrock/converse/{_HAIKU}", f"{wire.url}/bc/") + databricks: Final = _deploy( + scenario, "databricks/databricks-meta-llama-3-3-70b-instruct", f"{wire.url}/db/serving-endpoints/" + ) + embed_model: Final = _deploy(scenario, "mistral/mistral-embed", f"{wire.url}/mi/v1/") + + def openai_client() -> openai.AsyncOpenAI: + return openai.AsyncOpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) + + def chat(model: str) -> Call: + async def call(marker: str) -> Answer: + async with openai_client() as client: + completion: Final = await client.chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}] + ) + return completion.choices[0].message.content, completion.id + + return call + + def streamed(model: str) -> Call: + async def call(marker: str) -> Answer: + async with openai_client() as client: + stream: Final = await client.chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}], stream=True + ) + parts: Final = [chunk.choices[0].delta.content or "" async for chunk in stream if chunk.choices] + return "".join(parts), None + + return call + + async def messages(marker: str) -> Answer: + async with anthropic.AsyncAnthropic( + base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0 + ) as client: + message: Final = await client.messages.create( + model=claude, max_tokens=16, messages=[{"role": "user", "content": marker}] + ) + return "".join(block.text for block in message.content if block.type == "text"), None + + async def embed(marker: str) -> Answer: + async with httpx.AsyncClient(base_url=str(gateway.client.base_url), timeout=60, trust_env=False) as client: + response: Final = await client.post( + "/v1/embeddings", + json={"model": embed_model, "input": marker}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + data: Final = body_of(response)["data"] + vector_matches: Final = data == [{"object": "embedding", "index": 0, "embedding": list(VECTOR)}] + return (reply_text(marker) if vector_matches else str(data)), None + + kinds: Final[tuple[tuple[str, Call], ...]] = ( + ("/an/v1/messages", chat(claude)), + ("/an/v1/messages", streamed(claude)), + ("/an/v1/messages", messages), + (_GENERATE, chat(gemini)), + (_STREAM_GENERATE, streamed(gemini)), + ("/mi/v1/chat/completions", chat(mistral)), + ("/mi/v1/chat/completions", streamed(mistral)), + (f"/bc/model/{_HAIKU}/converse", chat(converse)), + (f"/bc/model/{_HAIKU}/converse-stream", streamed(converse)), + ("/db/serving-endpoints/chat/completions", chat(databricks)), + ("/mi/v1/embeddings", embed), + ) + repeated_kinds: Final = tuple(chain.from_iterable(repeat(kind, _PER_KIND) for kind in kinds)) + calls: Final = tuple((target, call, new_marker()) for target, call in repeated_kinds) + + async def burst() -> list[Answer]: + return await asyncio.gather(*(call(marker) for _, call, marker in calls)) + + answers: Final = asyncio.run(burst()) + assert [text for text, _ in answers] == [reply_text(marker) for *_, marker in calls] + received: Final = wire.drain() + assert len(received) == len(calls), sorted(request.target for request in received) + for target, _, marker in calls: + assert_one_request(of_marker(received, marker), target, marker) + logged: Final = { + identity: f"{wire.url}{target}" + for (target, _, _), (_, identity) in zip(calls, answers, strict=True) + if identity is not None + } + assert _spend_api_bases(tuple(logged)) == logged + + +def test_upstream_dropping_doomed_connections_mid_burst_fails_only_those_calls(gateway: Gateway) -> None: + markers: Final = tuple(new_marker() for _ in range(16)) + doomed: Final = frozenset(markers[::2]) + + def respond(request: Request) -> Reply: + return Reply(drop_connection=True) if marker_of(request) in doomed else join_peer(request) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + claude: Final = _deploy(scenario, "anthropic/claude-haiku-5-5", f"{wire.url}/an/") + gemini: Final = _deploy(scenario, "gemini/gemini-3.8-flash", f"{wire.url}/gm/v1beta/") + routed: Final = { + marker: (claude, "/an/v1/messages") if index % 4 < 2 else (gemini, _GENERATE) + for index, marker in enumerate(markers) + } + + async def call(marker: str) -> tuple[int, str]: + async with httpx.AsyncClient(base_url=str(gateway.client.base_url), timeout=60, trust_env=False) as client: + response: Final = await client.post( + "/v1/chat/completions", + json={"model": routed[marker][0], "messages": [{"role": "user", "content": marker}]}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) + return response.status_code, response.text + + async def burst() -> list[tuple[int, str]]: + return await asyncio.gather(*(call(marker) for marker in markers)) + + answers: Final = dict(zip(markers, asyncio.run(burst()), strict=True)) + received: Final = wire.drain() + assert_control_answers(gateway, scenario) + for marker in doomed: + assert answers[marker][0] >= 500 and "error" in answers[marker][1], answers[marker] + assert {request.target for request in of_marker(received, marker)} == {routed[marker][1]}, marker + for marker in frozenset(markers) - doomed: + assert answers[marker][0] == 200 and reply_text(marker) in answers[marker][1], answers[marker] + assert_one_request(of_marker(received, marker), routed[marker][1], marker) diff --git a/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py b/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py new file mode 100644 index 00000000000..4a075138305 --- /dev/null +++ b/tests/integration/providers/test_api_base_url_join_host_owned_proxy.py @@ -0,0 +1,140 @@ +import os +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Final + +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, gateway_from_environment +from integration._support.forward_proxy import ForwardProxy, refusing_forward_proxy +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration.providers._api_base_trailing_slash_support import KEY, new_marker +from pydantic import JsonValue + +_AZURE_HOST: Final = "anthropic-prod.services.ai.azure.com" +_EMBEDDINGS: Final = "host-embeddings" +_RERANK: Final = "host-rerank" +_AZURE_CLAUDE: Final = "host-azure-claude" +_AZURE_COUNT: Final = "host-azure-count" +_EMPTY_BASE: Final = "host-empty-base" +_SCRUBBED_PREFIXES: Final = ("ANTHROPIC_", "AZURE_AI_", "INFINITY_") +_PROXY_VARIABLES: Final = frozenset({"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY"}) +_load_yaml: Final[Callable[[str], object]] = yaml.safe_load + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + + +def _deployment(name: str, model: str, api_base: str) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, "api_key": KEY}} + + +def _config(directory: Path) -> Path: + base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text())) + configuration: Final = { + **base, + "model_list": [ + _deployment(_EMBEDDINGS, "infinity/BAAI/bge-small-en-v1.5", "http://embeddings"), + _deployment(_RERANK, "infinity/BAAI/bge-reranker-base", "http://rerank"), + _deployment(_AZURE_CLAUDE, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}/anthropic"), + _deployment(_AZURE_COUNT, "azure_ai/claude-haiku-5-5", f"http://{_AZURE_HOST}"), + _deployment(_EMPTY_BASE, "anthropic/claude-haiku-5-5", ""), + ], + "router_settings": {"disable_cooldowns": True, "num_retries": 0}, + } + path: Final = directory / "host-named-like-the-route.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +def _scrubbed() -> tuple[str, ...]: + return tuple( + name + for name in os.environ + if name.upper() in _PROXY_VARIABLES or name.upper().startswith(_SCRUBBED_PREFIXES) + ) + + +@pytest.fixture(scope="module") +def forward() -> Iterator[ForwardProxy]: + with refusing_forward_proxy() as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def isolated(forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("url-join-host") + with gateway_from_environment() as shared, owned_proxy( + shared, + directory, + { + "HTTP_PROXY": forward.url, + "HTTPS_PROXY": forward.url, + "NO_PROXY": "127.0.0.1,localhost", + "PROXY_BATCH_POLLING_ENABLED": "false", + }, + config=_config(directory), + remove_environment=_scrubbed(), + ) as candidate: + yield candidate + + +def _targets_after(forward: ForwardProxy, call: Callable[[], object]) -> frozenset[str]: + forward.drain() + call() + targets: Final = frozenset(forward.targets()) + assert targets, "the forward proxy saw no outbound request" + return targets + + +def _chat(isolated: Gateway, model: str) -> object: + return isolated.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": new_marker()}]} + ) + + +def _messages(isolated: Gateway, model: str, path: str) -> object: + return isolated.request( + "POST", + path, + {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": new_marker()}]}, + headers={"anthropic-version": "2023-06-01"}, + ) + + +def test_embedding_host_named_embeddings_gets_the_route_appended(isolated: Gateway, forward: ForwardProxy) -> None: + targets: Final = _targets_after( + forward, lambda: isolated.request("POST", "/v1/embeddings", {"model": _EMBEDDINGS, "input": new_marker()}) + ) + assert targets == {"http://embeddings/embeddings"} + + +def test_rerank_host_named_rerank_gets_the_route_appended(isolated: Gateway, forward: ForwardProxy) -> None: + body: Final[dict[str, JsonValue]] = {"model": _RERANK, "query": new_marker(), "documents": ["only document"]} + targets: Final = _targets_after(forward, lambda: isolated.request("POST", "/v1/rerank", body)) + assert targets == {"http://rerank/rerank"} + + +def test_azure_claude_chat_on_a_host_starting_with_anthropic_keeps_the_host( + isolated: Gateway, forward: ForwardProxy +) -> None: + targets: Final = _targets_after(forward, lambda: _chat(isolated, _AZURE_CLAUDE)) + assert targets == {f"http://{_AZURE_HOST}/anthropic/v1/messages"} + + +def test_azure_claude_messages_on_a_host_starting_with_anthropic_keeps_the_host( + isolated: Gateway, forward: ForwardProxy +) -> None: + targets: Final = _targets_after(forward, lambda: _messages(isolated, _AZURE_CLAUDE, "/v1/messages")) + assert targets == {f"http://{_AZURE_HOST}/anthropic/v1/messages"} + + +def test_azure_claude_count_tokens_on_a_bare_host_starting_with_anthropic_adds_the_anthropic_path( + isolated: Gateway, forward: ForwardProxy +) -> None: + targets: Final = _targets_after(forward, lambda: _messages(isolated, _AZURE_COUNT, "/v1/messages/count_tokens")) + assert targets == {f"http://{_AZURE_HOST}/anthropic/v1/messages/count_tokens"} + + +def test_empty_api_base_falls_back_to_the_provider_default_host(isolated: Gateway, forward: ForwardProxy) -> None: + targets: Final = _targets_after(forward, lambda: _chat(isolated, _EMPTY_BASE)) + assert targets == {"api.anthropic.com:443"} diff --git a/tests/integration/providers/test_api_base_url_join_wire.py b/tests/integration/providers/test_api_base_url_join_wire.py new file mode 100644 index 00000000000..42e53ff9f95 --- /dev/null +++ b/tests/integration/providers/test_api_base_url_join_wire.py @@ -0,0 +1,373 @@ +import json +from typing import Final +from urllib.parse import urlsplit + +import anthropic +import openai +import pytest +from integration._support.client import Gateway, Scenario, eventually, list_value, object_value, string_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server +from integration.providers._api_base_trailing_slash_support import ( + KEY, + TRAILING, + TRAILING_IDS, + VECTOR, + Row, + assert_control_answers, + assert_one_request, + body_of, + first_item, + new_marker, + reply_text, + request_json, +) +from integration.providers._api_base_url_join_support import COUNTED_TOKENS, RELEVANCE, join_peer +from pydantic import JsonValue + +_HAIKU: Final = "anthropic.claude-haiku-5-5" +_ANTHROPIC: Final = Row("anthropic/claude-haiku-5-5", "/an", "/v1/messages") +_GEMINI: Final = Row("gemini/gemini-3.8-flash", "/gm/v1beta", "/models/gemini-3.8-flash:generateContent") +_MISTRAL: Final = Row("mistral/mistral-small-latest", "/mi/v1", "/chat/completions") +_BEDROCK_INVOKE: Final = Row(f"bedrock/invoke/{_HAIKU}", "/bi", f"/model/{_HAIKU}/invoke") +_CHAT_ROWS: Final = ( + _ANTHROPIC, + _GEMINI, + _MISTRAL, + Row("databricks/databricks-meta-llama-3-3-70b-instruct", "/db/serving-endpoints", "/chat/completions"), + _BEDROCK_INVOKE, + Row(f"bedrock/converse/{_HAIKU}", "/bc", f"/model/{_HAIKU}/converse"), +) +_STREAM_ROWS: Final = ( + _ANTHROPIC, + Row("gemini/gemini-3.8-flash", "/gm/v1beta", "/models/gemini-3.8-flash:streamGenerateContent?alt=sse"), + _MISTRAL, + Row(f"bedrock/converse/{_HAIKU}", "/bc", f"/model/{_HAIKU}/converse-stream"), +) +_MESSAGES_ROWS: Final = (_ANTHROPIC, _BEDROCK_INVOKE, _GEMINI, _MISTRAL) +_RESPONSES_ROWS: Final = (_ANTHROPIC, _GEMINI) +_EMBEDDING_ROWS: Final = ( + Row("mistral/mistral-embed", "/mi/v1", "/embeddings"), + Row("gemini/gemini-embedding-2", "/gm/v1beta", "/models/gemini-embedding-2:batchEmbedContents"), +) +_TRANSCRIPTION: Final = Row("mistral/voxtral-mini-latest", "/mi/v1", "/audio/transcriptions") +_FULL_ROUTE_SUFFIXES: Final = ("", "/", "?beta=true") +_FULL_ROUTE_IDS: Final = ("exact", "slash", "query") + + +def _ids(rows: tuple[Row, ...]) -> tuple[str, ...]: + return tuple(row.model for row in rows) + + +def _deploy(scenario: Scenario, row: Row, api_base: str) -> str: + if row.model.startswith("bedrock/"): + return scenario.model(model=row.model, api_base=api_base, api_key=KEY, aws_region_name="us-east-1") + return scenario.model(model=row.model, api_base=api_base, api_key=KEY) + + +def _openai(gateway: Gateway) -> openai.OpenAI: + return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) + + +def _anthropic(gateway: Gateway) -> anthropic.Anthropic: + return anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0) + + +def _chat(gateway: Gateway, model: str, marker: str) -> str | None: + completion: Final = _openai(gateway).chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}] + ) + return completion.choices[0].message.content + + +def _message_text(gateway: Gateway, model: str, marker: str) -> list[str]: + message: Final = _anthropic(gateway).messages.create( + model=model, max_tokens=16, messages=[{"role": "user", "content": marker}] + ) + return [block.text for block in message.content if block.type == "text"] + + +def _spend_rows(pattern: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT request_id, api_base, cache_hit FROM "LiteLLM_SpendLogs" WHERE request_id LIKE %s', (pattern,) + ) + + +def _one_spend_row(pattern: str) -> dict[str, JsonValue]: + (row,) = eventually(lambda: _spend_rows(pattern), lambda rows: len(rows) == 1, seconds=70) + return row + + +def _gemini_contents(marker: str) -> dict[str, JsonValue]: + return {"contents": [{"role": "user", "parts": [{"text": marker}]}]} + + +def _gemini_text(payload: dict[str, JsonValue]) -> JsonValue: + content: Final = object_value(first_item(payload["candidates"])["content"]) + return first_item(content["parts"])["text"] + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +@pytest.mark.parametrize("row", _CHAT_ROWS, ids=_ids(_CHAT_ROWS)) +def test_chat_completion_reaches_one_single_slash_url(gateway: Gateway, row: Row, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, row, f"{wire.url}{row.base}{trailing}") + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), row.target, marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +@pytest.mark.parametrize("row", _STREAM_ROWS, ids=_ids(_STREAM_ROWS)) +def test_streamed_chat_reaches_one_single_slash_url(gateway: Gateway, row: Row, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, row, f"{wire.url}{row.base}{trailing}") + stream: Final = _openai(gateway).chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}], stream=True + ) + text: Final = "".join(chunk.choices[0].delta.content or "" for chunk in stream if chunk.choices) + assert text == reply_text(marker) + assert_one_request(wire.drain(), row.target, marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +@pytest.mark.parametrize("row", _MESSAGES_ROWS, ids=_ids(_MESSAGES_ROWS)) +def test_messages_reaches_one_single_slash_url(gateway: Gateway, row: Row, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, row, f"{wire.url}{row.base}{trailing}") + assert _message_text(gateway, model, marker) == [reply_text(marker)] + assert_one_request(wire.drain(), row.target, marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_streamed_messages_reach_one_single_slash_url(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an{trailing}") + with _anthropic(gateway).messages.stream( + model=model, max_tokens=16, messages=[{"role": "user", "content": marker}] + ) as stream: + text: Final = "".join(stream.text_stream) + assert text == reply_text(marker) + assert request_json(assert_one_request(wire.drain(), "/an/v1/messages", marker))["stream"] is True + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +@pytest.mark.parametrize("row", _RESPONSES_ROWS, ids=_ids(_RESPONSES_ROWS)) +def test_responses_bridge_reaches_one_single_slash_url(gateway: Gateway, row: Row, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, row, f"{wire.url}{row.base}{trailing}") + assert _openai(gateway).responses.create(model=model, input=marker).output_text == reply_text(marker) + assert_one_request(wire.drain(), row.target, marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +@pytest.mark.parametrize("row", _EMBEDDING_ROWS, ids=_ids(_EMBEDDING_ROWS)) +def test_embedding_reaches_one_single_slash_url(gateway: Gateway, row: Row, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, row, f"{wire.url}{row.base}{trailing}") + response: Final = gateway.request("POST", "/v1/embeddings", {"model": model, "input": marker}) + assert first_item(body_of(response)["data"])["embedding"] == list(VECTOR) + assert_one_request(wire.drain(), row.target, marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_mistral_transcription_reaches_one_single_slash_url(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _TRANSCRIPTION, f"{wire.url}/mi/v1{trailing}") + response: Final = gateway.request_multipart( + "/v1/audio/transcriptions", + {"model": model}, + {"file": ("speech.wav", f"RIFF {marker}".encode(), "audio/wav")}, + ) + assert body_of(response)["text"] == reply_text(marker) + assert_one_request(wire.drain(), _TRANSCRIPTION.target, marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_google_genai_generate_content_reaches_one_single_slash_url(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _GEMINI, f"{wire.url}/gm/v1beta{trailing}") + response: Final = gateway.request( + "POST", f"/v1beta/models/{model}:generateContent", _gemini_contents(marker) + ) + assert _gemini_text(body_of(response)) == reply_text(marker) + assert_one_request(wire.drain(), _GEMINI.target, marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_google_genai_stream_generate_content_reaches_one_single_slash_url(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _GEMINI, f"{wire.url}/gm/v1beta{trailing}") + response: Final = gateway.request( + "POST", + f"/v1beta/models/{model}:streamGenerateContent", + _gemini_contents(marker), + params={"alt": "sse"}, + ) + assert response.status_code == 200, response.text + assert marker in response.text, response.text + assert_one_request( + wire.drain(), "/gm/v1beta/models/gemini-3.8-flash:streamGenerateContent?alt=sse", marker + ) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_count_tokens_reaches_one_single_slash_url(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an{trailing}") + response: Final = gateway.request( + "POST", "/v1/messages/count_tokens", {"model": model, "messages": [{"role": "user", "content": marker}]} + ) + assert body_of(response)["input_tokens"] == COUNTED_TOKENS + assert_one_request(wire.drain(), "/an/v1/messages/count_tokens", marker) + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_health_check_reaches_one_single_slash_url(gateway: Gateway, trailing: str) -> None: + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an{trailing}") + report: Final = body_of(gateway.request("GET", "/health", params={"model": model})) + assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report + assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/an/v1/messages")] + + +@pytest.mark.parametrize("trailing", TRAILING, ids=TRAILING_IDS) +def test_spend_log_records_the_single_slash_url_it_called(gateway: Gateway, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an{trailing}") + response: Final = gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]} + ) + identity: Final = string_value(body_of(response)["id"]) + assert_one_request(wire.drain(), "/an/v1/messages", marker) + assert _one_spend_row(identity)["api_base"] == f"{wire.url}/an/v1/messages" + + +def test_cache_hit_on_a_slash_base_skips_the_upstream_and_logs_its_own_row(gateway: Gateway) -> None: + pytest.skip("BUG: a response-cache hit writes two cache_hit spend rows for one chat completion") + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/") + body: Final[dict[str, JsonValue]] = {"model": model, "messages": [{"role": "user", "content": marker}]} + first: Final = body_of(gateway.request("POST", "/v1/chat/completions", body)) + second: Final = body_of(gateway.request("POST", "/v1/chat/completions", body)) + assert first_item(second["choices"])["message"] == first_item(first["choices"])["message"] + assert_one_request(wire.drain(), "/an/v1/messages", marker) + identity: Final = string_value(first["id"]) + assert _one_spend_row(identity)["api_base"] == f"{wire.url}/an/v1/messages" + hit: Final = _one_spend_row(f"{identity}_cache_hit%") + assert hit["cache_hit"] == "True", hit + assert "//" not in urlsplit(string_value(hit["api_base"])).path, hit + + +@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) +def test_chat_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}") + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker) + + +@pytest.mark.parametrize("suffix", _FULL_ROUTE_SUFFIXES, ids=_FULL_ROUTE_IDS) +def test_messages_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/v1/messages{suffix}") + assert _message_text(gateway, model, marker) == [reply_text(marker)] + assert_one_request(wire.drain(), f"/an/v1/messages{suffix.rstrip('/')}", marker) + + +@pytest.mark.parametrize("suffix", ("", "/", "?tier=a"), ids=("exact", "slash", "query")) +def test_rerank_base_that_already_names_the_route_keeps_its_path_and_query(gateway: Gateway, suffix: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, Row("cohere/rerank-v3.5", "/co", ""), f"{wire.url}/co/v2/rerank{suffix}") + response: Final = gateway.request( + "POST", "/v1/rerank", {"model": model, "query": marker, "documents": ["only document"]} + ) + assert [object_value(result)["relevance_score"] for result in list_value(body_of(response)["results"])] == [ + RELEVANCE + ] + assert_one_request(wire.drain(), f"/co/v2/rerank{suffix.rstrip('/')}", marker) + + +@pytest.mark.parametrize( + ("row", "trailing"), + ( + (_ANTHROPIC, "//"), + (Row("anthropic/claude-haiku-5-5", f"/an/{'p' * 5000}", "/v1/messages"), "/"), + (_GEMINI, "//"), + (Row("gemini/gemini-3.8-flash", f"/gm/{'p' * 5000}", "/models/gemini-3.8-flash:generateContent"), "/"), + ), + ids=("anthropic-two-slashes", "anthropic-5kb-prefix", "gemini-two-slashes", "gemini-5kb-prefix"), +) +def test_base_with_repeated_slashes_or_a_long_prefix_reaches_one_url(gateway: Gateway, row: Row, trailing: str) -> None: + marker: Final = new_marker() + with wire_server(join_peer) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, row, f"{wire.url}{row.base}{trailing}") + assert _chat(gateway, model, marker) == reply_text(marker) + assert_one_request(wire.drain(), row.target, marker) + + +def _anthropic_error(status: int, marker: str) -> bytes: + return json.dumps( + {"type": "error", "error": {"type": "authentication_error", "message": f"scripted {status} {marker}"}} + ).encode() + + +def _gemini_error(status: int, marker: str) -> bytes: + error: Final = {"code": status, "message": f"scripted {status} {marker}", "status": "NOT_FOUND"} + return json.dumps({"error": error}).encode() + + +@pytest.mark.parametrize( + ("row", "status"), + ((_ANTHROPIC, 401), (_GEMINI, 404)), + ids=("anthropic-401", "gemini-404"), +) +def test_upstream_error_on_a_slash_base_reaches_the_caller_from_one_url( + gateway: Gateway, row: Row, status: int +) -> None: + marker: Final = new_marker() + error: Final = _anthropic_error(status, marker) if row is _ANTHROPIC else _gemini_error(status, marker) + + def respond(_: Request) -> Reply: + return Reply(status=status, body=error) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, row, f"{wire.url}{row.base}/") + response: Final = gateway.request( + "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": marker}]} + ) + assert response.status_code == status, response.text + assert marker in response.text, response.text + assert_one_request(wire.drain(), row.target, marker) + assert_control_answers(gateway, scenario) + + +def test_streamed_chat_upstream_error_on_a_slash_base_reaches_the_caller(gateway: Gateway) -> None: + marker: Final = new_marker() + + def respond(_: Request) -> Reply: + return Reply(status=401, body=_anthropic_error(401, marker)) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = _deploy(scenario, _ANTHROPIC, f"{wire.url}/an/") + with pytest.raises(openai.AuthenticationError) as raised: + _openai(gateway).chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}], stream=True + ) + assert marker in str(raised.value) + assert_one_request(wire.drain(), "/an/v1/messages", marker) + assert_control_answers(gateway, scenario) diff --git a/tests/unit/litellm_core_utils/test_url_utils.py b/tests/unit/litellm_core_utils/test_url_utils.py index 46bb91959c0..9b29eac208e 100644 --- a/tests/unit/litellm_core_utils/test_url_utils.py +++ b/tests/unit/litellm_core_utils/test_url_utils.py @@ -1,6 +1,8 @@ import asyncio import socket import threading +from collections.abc import Callable +from urllib.parse import urlsplit import httpx import pytest @@ -13,6 +15,8 @@ from litellm.litellm_core_utils.url_utils import ( assert_same_origin, encode_url_path_segment, encode_url_path_segments, + ensure_api_route, + join_api_base, validate_url, ) @@ -571,3 +575,80 @@ async def test_async_safe_get_resolves_dns_off_the_event_loop(monkeypatch): assert response.status_code == 200 assert resolver_threads and all(thread is not loop_thread for thread in resolver_threads) assert released_by_the_loop and all(released_by_the_loop) + + +@pytest.mark.parametrize( + ("api_base", "route", "expected"), + [ + ("https://gw.test/p/", "v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/p//", "v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/p", "v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/p", "/v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/p/", "/v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/p/v1/messages", "v1/messages", "https://gw.test/p/v1/messages/v1/messages"), + ("https://api.test/v1/evals/cancel", "cancel", "https://api.test/v1/evals/cancel/cancel"), + ("http://embeddings", "embeddings", "http://embeddings/embeddings"), + ("http:/", "v1/files", "http://v1/files"), + ("https://gw.test/p/?x=1", "v1/messages", "https://gw.test/p?x=1/v1/messages"), + ], + ids=( + "trailing-slash-base", + "repeated-trailing-slashes", + "base-without-slash", + "route-with-leading-slash", + "slashes-on-both-sides", + "base-already-ends-in-route", + "id-equal-to-the-route", + "host-named-like-the-route", + "host-stripped-as-a-suffix-leaves-only-the-scheme", + "slash-before-the-query-matches-the-base-without-it", + ), +) +def test_join_api_base_always_appends_the_route_after_exactly_one_slash( + api_base: str, route: str, expected: str +) -> None: + assert join_api_base(api_base, route) == expected + + +@pytest.mark.parametrize("api_base", ["https://", "http://", "https:///"]) +@pytest.mark.parametrize("route", ["v1/messages", "models/gemini-flash:generateContent"]) +@pytest.mark.parametrize("join", [join_api_base, ensure_api_route], ids=["join_api_base", "ensure_api_route"]) +def test_a_base_that_is_only_a_scheme_never_turns_the_route_into_the_host( + api_base: str, route: str, join: Callable[[str, str], str] +) -> None: + assert urlsplit(join(api_base, route)).netloc == "" + + +@pytest.mark.parametrize( + ("api_base", "route", "expected"), + [ + ("https://gw.test/p/v1/messages", "v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/p/v1/messages/", "/v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/x/listen", "listen", "https://gw.test/x/listen"), + ("https://gw.test/p/", "v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/p", "/v1/messages", "https://gw.test/p/v1/messages"), + ("https://gw.test/research", "search", "https://gw.test/research/search"), + ("http://embeddings", "embeddings", "http://embeddings/embeddings"), + ("http://embeddings:8080/", "embeddings", "http://embeddings:8080/embeddings"), + ("http:/", "anthropic/v1/messages", "http://anthropic/v1/messages"), + ("https://gw.test/p/v1/messages/?beta=true", "v1/messages", "https://gw.test/p/v1/messages?beta=true"), + ("https://gw.test/p/v1/messages?beta=true", "v1/messages", "https://gw.test/p/v1/messages?beta=true"), + ], + ids=( + "path-already-ends-in-route", + "path-ends-in-route-before-trailing-slash", + "last-segment-is-the-route", + "trailing-slash-base", + "route-with-leading-slash", + "path-ends-in-lookalike-suffix", + "host-named-like-the-route", + "host-with-port-named-like-the-route", + "host-stripped-as-a-suffix-leaves-only-the-scheme", + "route-path-with-a-slash-before-the-query", + "route-path-before-the-query", + ), +) +def test_ensure_api_route_appends_the_route_only_when_the_url_path_lacks_it( + api_base: str, route: str, expected: str +) -> None: + assert ensure_api_route(api_base, route) == expected diff --git a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py index 5c1ff1b46a3..f1db76c1e21 100644 --- a/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py +++ b/tests/unit/llms/azure/realtime/test_azure_realtime_handler.py @@ -740,3 +740,15 @@ async def test_arealtime_forwards_deployment_azure_ad_token(monkeypatch, no_ambi ) assert mock_async_realtime.call_args.kwargs["azure_ad_token"] == "deployment-entra-token" + + +@pytest.mark.parametrize("protocol", [None, "GA"]) +def test_construct_url_with_a_trailing_slash_api_base_matches_the_url_without_it(protocol): + from litellm.llms.azure.realtime.handler import AzureOpenAIRealtime + + def url_for(api_base: str) -> str: + return AzureOpenAIRealtime().construct_url( + api_base=api_base, model="gpt-realtime", api_version="2024-10-01-preview", realtime_protocol=protocol + ) + + assert url_for("https://gw.example.com/azure/") == url_for("https://gw.example.com/azure") diff --git a/tests/unit/llms/azure/test_azure_common_utils.py b/tests/unit/llms/azure/test_azure_common_utils.py index b3bd1b8743a..1dc7ff74323 100644 --- a/tests/unit/llms/azure/test_azure_common_utils.py +++ b/tests/unit/llms/azure/test_azure_common_utils.py @@ -1702,6 +1702,24 @@ def test_azure_v1_api_uses_openai_client(api_version): ), f"base_url should contain /openai/v1/, got {async_client.base_url}" +@pytest.mark.parametrize( + "api_base", + [ + "https://test.openai.azure.com", + "https://test.openai.azure.com/", + "https://test.openai.azure.com/openai/v1", + "https://test.openai.azure.com/openai/v1/", + ], +) +@pytest.mark.parametrize("is_async", [False, True]) +def test_azure_v1_api_client_targets_openai_v1_once_whatever_the_api_base_ends_in(api_base: str, is_async: bool): + client: Final = BaseAzureLLM().get_azure_openai_client( + api_key="test-key", api_base=api_base, api_version="v1", _is_async=is_async + ) + + assert str(client.base_url) == "https://test.openai.azure.com/openai/v1/" + + @pytest.mark.parametrize("api_version", ["v1", "latest", "preview"]) def test_azure_v1_api_with_azure_ad_token_provider(api_version): """ diff --git a/tests/unit/llms/azure_ai/agents/test_handler.py b/tests/unit/llms/azure_ai/agents/test_handler.py index f92f5359c49..71466bb172c 100644 --- a/tests/unit/llms/azure_ai/agents/test_handler.py +++ b/tests/unit/llms/azure_ai/agents/test_handler.py @@ -22,10 +22,10 @@ def _httpx_only_transport(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") -def _assistants_api(thread_id: str, answer: str) -> respx.MockRouter: - thread_path: Final = f"/api/projects/litellm-test/threads/{thread_id}" +def _assistants_api(thread_id: str, answer: str, project_path: str = "/api/projects/litellm-test") -> respx.MockRouter: + thread_path: Final = f"{project_path}/threads/{thread_id}" router: Final = respx.mock(assert_all_called=False) - router.post(path="/api/projects/litellm-test/threads").respond( + router.post(path=f"{project_path}/threads").respond( 200, json={"id": thread_id, "object": "thread", "created_at": 1767225600, "metadata": {}} ) router.post(path=f"{thread_path}/messages").respond( @@ -116,6 +116,23 @@ async def test_acompletion_creates_thread_posts_message_runs_and_returns_the_rep assert response._hidden_params["thread_id"] == "thread_new" +@pytest.mark.asyncio +async def test_acompletion_creates_the_thread_under_a_project_named_threads() -> None: + router: Final = _assistants_api("thread_new", "hello", project_path="/api/projects/threads") + + with router: + await litellm.acompletion( + model=f"azure_ai/agents/{AGENT_ID}", + messages=[{"role": "user", "content": "hi"}], + api_base="https://agents-test.services.ai.azure.com/api/projects/threads", + api_key="azure-ad-token", + stream=False, + ) + recorded: Final = _recorded(router) + + assert recorded[0][:2] == ("POST", f"/api/projects/threads/threads?api-version={API_VERSION}") + + @pytest.mark.asyncio async def test_handler_acompletion_with_thread_id_reuses_that_thread() -> None: router: Final = _assistants_api("thread_existing", "Your name is Alice.") diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py index 4b317cff975..8971bd31a60 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_count_tokens_transformation.py @@ -4,8 +4,9 @@ Tests for Azure AI Anthropic CountTokens transformation. Verifies that the CountTokens API uses the correct authentication headers. """ +from typing import Final - +import pytest from litellm.llms.azure_ai.anthropic.count_tokens.transformation import ( AzureAIAnthropicCountTokensConfig, @@ -105,3 +106,22 @@ class TestAzureAIAnthropicCountTokensConfig: endpoint == "https://my-resource.services.ai.azure.com/anthropic/v1/messages/count_tokens" ) + + def test_get_count_tokens_endpoint_for_a_resource_named_anthropic(self): + config: Final = AzureAIAnthropicCountTokensConfig() + + endpoint: Final = config.get_count_tokens_endpoint("https://anthropic-res.services.ai.azure.com") + + assert endpoint == "https://anthropic-res.services.ai.azure.com/anthropic/v1/messages/count_tokens" + + @pytest.mark.parametrize( + "api_base", + [ + "https://my-resource.services.ai.azure.com/anthropic/v1/messages", + "https://my-resource.services.ai.azure.com/anthropic/v1/messages/", + ], + ) + def test_get_count_tokens_endpoint_sits_next_to_the_messages_url(self, api_base: str): + endpoint: Final = AzureAIAnthropicCountTokensConfig().get_count_tokens_endpoint(api_base) + + assert endpoint == "https://my-resource.services.ai.azure.com/anthropic/v1/messages/count_tokens" diff --git a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py index 38483b8a1fa..5e0395f84dc 100644 --- a/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py +++ b/tests/unit/llms/azure_ai/claude/test_azure_anthropic_messages_transformation.py @@ -198,6 +198,17 @@ class TestAzureAnthropicMessagesConfig: assert url == "https://test.services.ai.azure.com/anthropic/v1/messages" + def test_get_complete_url_adds_the_anthropic_path_for_a_host_named_anthropic(self): + url: Final = AzureAnthropicMessagesConfig().get_complete_url( + api_base="https://anthropic-res.services.ai.azure.com", + api_key="test-api-key", + model="claude-sonnet-4-5", + optional_params={}, + litellm_params={}, + ) + + assert url == "https://anthropic-res.services.ai.azure.com/anthropic/v1/messages" + def test_get_complete_url_raises_error_when_api_base_missing(self): """Test get_complete_url raises error when api_base is None""" config = AzureAnthropicMessagesConfig() diff --git a/tests/unit/llms/bedrock/chat/agentcore/test_agentcore_transformation.py b/tests/unit/llms/bedrock/chat/agentcore/test_agentcore_transformation.py index 053ba477212..413061a732d 100644 --- a/tests/unit/llms/bedrock/chat/agentcore/test_agentcore_transformation.py +++ b/tests/unit/llms/bedrock/chat/agentcore/test_agentcore_transformation.py @@ -1269,3 +1269,17 @@ def test_agentcore_runtime_response_becomes_assistant_message( assert json.loads(route.calls[0].request.content)["prompt"] == "Explain machine learning in simple terms" assert response.choices[0].message.content == "Machine learning lets computers learn from data." assert response.choices[0].finish_reason == "stop" + + +def test_get_complete_url_sends_a_non_string_runtime_endpoint_to_that_endpoint(): + vpc_endpoint: Final = "https://vpce-1.bedrock-agentcore.us-west-2.vpce.amazonaws.com" + + url: Final = AmazonAgentCoreConfig().get_complete_url( + api_base=None, + api_key=None, + model="agentcore/arn:aws:bedrock-agentcore:us-west-2:111122223333:runtime/my-agent", + optional_params={"aws_bedrock_runtime_endpoint": httpx.URL(f"{vpc_endpoint}/")}, + litellm_params={}, + ) + + assert url.startswith(f"{vpc_endpoint}/runtimes/") diff --git a/tests/unit/llms/openai/evals/test_openai_evals_transformation.py b/tests/unit/llms/openai/evals/test_openai_evals_transformation.py index f359f214f97..ed8f86d4727 100644 --- a/tests/unit/llms/openai/evals/test_openai_evals_transformation.py +++ b/tests/unit/llms/openai/evals/test_openai_evals_transformation.py @@ -264,6 +264,17 @@ def test_transform_cancel_eval_request(config: OpenAIEvalsConfig): assert request_body == {} +def test_transform_cancel_eval_request_appends_cancel_to_an_eval_id_named_cancel(config: OpenAIEvalsConfig) -> None: + eval_url, _ = config.transform_get_eval_request( + eval_id="cancel", api_base="https://api.openai.com", litellm_params=GenericLiteLLMParams(), headers={} + ) + cancel_url, _, _ = config.transform_cancel_eval_request( + eval_id="cancel", api_base="https://api.openai.com", litellm_params=GenericLiteLLMParams(), headers={} + ) + + assert cancel_url == f"{eval_url}/cancel" + + def test_transform_cancel_eval_response(config: OpenAIEvalsConfig): """Test transformation of cancel eval response""" response = httpx.Response( diff --git a/tests/unit/llms/test_api_base_trailing_slash_routes.py b/tests/unit/llms/test_api_base_trailing_slash_routes.py new file mode 100644 index 00000000000..ce8cecf11a5 --- /dev/null +++ b/tests/unit/llms/test_api_base_trailing_slash_routes.py @@ -0,0 +1,119 @@ +import asyncio +import contextlib +from collections.abc import Callable, Iterator, Mapping +from types import MappingProxyType +from typing import Final + +import pytest +import respx + +import litellm +from litellm.evals.main import cancel_run, create_run, get_run + +FAKE_KEY: Final = "sk-fake" +FAKE_AWS: Final[Mapping[str, str]] = MappingProxyType( + {"aws_access_key_id": "AKIAFAKE", "aws_secret_access_key": "fake-secret", "aws_region_name": "us-east-1"} +) +MESSAGES: Final = ({"role": "user", "content": "hi"},) +GATEWAY_PATH: Final = "/gw/" + + +GATEWAY_HOST: Final = "gateway.test" + + +@pytest.fixture +def received_paths() -> Iterator[Callable[[], tuple[str, ...]]]: + with respx.mock(assert_all_called=False) as router: + route: Final = router.route(host=GATEWAY_HOST).respond( + 404, json={"error": {"message": "not found", "code": 404}} + ) + yield lambda: tuple(call.request.url.raw_path.decode().split("?", 1)[0] for call in route.calls) + + +def _drain(result: object) -> None: + if isinstance(result, litellm.CustomStreamWrapper): + for _ in result: + pass + + +def _completion(model: str, stream: bool = False, **kwargs: object) -> Callable[[str], object]: + return lambda api_base: _drain( + litellm.completion(model=model, messages=list(MESSAGES), api_base=api_base, stream=stream, timeout=5, **kwargs) + ) + + +def _embedding(model: str, **kwargs: object) -> Callable[[str], object]: + return lambda api_base: litellm.embedding(model=model, input=["hi"], api_base=api_base, timeout=5, **kwargs) + + +ROUTES: Final[Mapping[str, Callable[[str], object]]] = MappingProxyType( + { + "anthropic-chat": _completion("anthropic/claude-opus-5-5", api_key=FAKE_KEY), + "bedrock-converse": _completion("bedrock/converse/anthropic.claude-sonnet-5-5", **FAKE_AWS), + "bedrock-converse-stream": _completion("bedrock/converse/anthropic.claude-sonnet-5-5", stream=True, **FAKE_AWS), + "gemini-chat": _completion("gemini/gemini-pro-latest", api_key=FAKE_KEY), + "gemini-chat-stream": _completion("gemini/gemini-pro-latest", stream=True, api_key=FAKE_KEY), + "mistral-chat": _completion("mistral/mistral-large-latest", api_key=FAKE_KEY), + "gigachat-chat": _completion("gigachat/GigaChat-2-Max", api_key=FAKE_KEY), + "bedrock-titan-embedding": _embedding("bedrock/amazon.titan-embed-text-v2:0", **FAKE_AWS), + "gemini-embedding": _embedding("gemini/gemini-embedding-2", api_key=FAKE_KEY), + "gigachat-embedding": _embedding("gigachat/Embeddings-2", api_key=FAKE_KEY), + "volcengine-embedding": _embedding("volcengine/doubao-embedding-text-240715", api_key=FAKE_KEY), + "ollama-embedding": _embedding("ollama/nomic-embed-text"), + "oobabooga-chat": _completion("oobabooga/local-model"), + "oobabooga-embedding": _embedding("oobabooga/local-model"), + "bedrock-rerank": lambda api_base: litellm.rerank( + model="bedrock/arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0", + query="hi", + documents=["a"], + api_base=api_base, + **FAKE_AWS, + ), + "bedrock-image-generation": lambda api_base: litellm.image_generation( + model="bedrock/amazon.nova-canvas-v1:0", prompt="a cat", api_base=api_base, timeout=5, **FAKE_AWS + ), + "gemini-interaction-get": lambda api_base: litellm.interactions.get( + interaction_id="interaction-1", api_base=api_base, api_key=FAKE_KEY + ), + "gemini-interaction-delete": lambda api_base: litellm.interactions.delete( + interaction_id="interaction-1", api_base=api_base, api_key=FAKE_KEY + ), + "gemini-interaction-cancel": lambda api_base: litellm.interactions.cancel( + interaction_id="interaction-1", api_base=api_base, api_key=FAKE_KEY + ), + "openai-eval-run-create": lambda api_base: create_run( + eval_id="eval-1", data_source={"type": "jsonl"}, api_base=api_base, api_key=FAKE_KEY + ), + "openai-eval-run-get": lambda api_base: get_run( + eval_id="eval-1", run_id="run-1", api_base=api_base, api_key=FAKE_KEY + ), + "openai-eval-run-cancel": lambda api_base: cancel_run( + eval_id="eval-1", run_id="run-1", api_base=api_base, api_key=FAKE_KEY + ), + "edenai-model-list": lambda api_base: litellm.get_valid_models( + check_provider_endpoint=True, custom_llm_provider="edenai", api_key=FAKE_KEY, api_base=api_base + ), + "gemini-rag-ingest": lambda api_base: asyncio.run( + litellm.aingest( + ingest_options={ + "vector_store": {"custom_llm_provider": "gemini", "api_base": api_base, "api_key": FAKE_KEY} + }, + file_data=("notes.txt", b"hello", "text/plain"), + ) + ), + } +) + + +@pytest.mark.parametrize("route", ROUTES.values(), ids=ROUTES.keys()) +def test_api_base_with_a_trailing_slash_reaches_the_gateway_path_with_single_slashes( + received_paths: Callable[[], tuple[str, ...]], route: Callable[[str], object], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("GIGACHAT_ACCESS_TOKEN", FAKE_KEY) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with contextlib.suppress(Exception): + route(f"http://{GATEWAY_HOST}{GATEWAY_PATH}") + + paths: Final = received_paths() + assert paths, "the route never reached the api_base" + assert all(path.startswith(GATEWAY_PATH) and "//" not in path for path in paths), paths diff --git a/tests/unit/llms/test_get_complete_url_trailing_slash.py b/tests/unit/llms/test_get_complete_url_trailing_slash.py new file mode 100644 index 00000000000..9118498c2c2 --- /dev/null +++ b/tests/unit/llms/test_get_complete_url_trailing_slash.py @@ -0,0 +1,278 @@ +import copy +import importlib +import inspect +import itertools +from collections.abc import Mapping +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType, ModuleType +from typing import Final +from urllib.parse import urlsplit, urlunsplit + +import pytest +from _pytest.mark.structures import ParameterSet + +import litellm +from litellm.llms.elevenlabs.text_to_speech.transformation import ElevenLabsTextToSpeechConfig +from litellm.llms.meta.realtime.transformation import MUSE_MODEL +from litellm.llms.openai_like.dynamic_config import create_config_class, create_responses_config_class +from litellm.llms.openai_like.json_loader import SimpleProviderConfig + +API_BASE_WITH_TRAILING_SLASH: Final = "https://gw.example.com/p/" +API_BASE_WITHOUT_TRAILING_SLASH: Final = API_BASE_WITH_TRAILING_SLASH.rstrip("/") +LLMS_ROOT: Final = Path(litellm.__file__).parent / "llms" +PACKAGE_ROOT: Final = LLMS_ROOT.parent.parent + +ARGUMENTS_BY_NAME: Final[Mapping[str, object]] = MappingProxyType( + { + "api_key": "sk-test", + "model": "test-model", + "optional_params": {}, + "litellm_params": {"api_key": "sk-test", "vertex_project": "project", "vertex_location": "us-central1"}, + "stream": False, + "endpoint": "v1/messages", + "request_query_params": None, + "data": None, + "api_version": "2024-01-01", + "vector_store_id": "vs_test", + } +) + + +SAMPLE_JSON_PROVIDER: Final = SimpleProviderConfig( + "json-provider", {"base_url": "https://json-provider.example.com", "api_key_env": "UNSET_KEY"} +) +FACTORY_BUILDERS: Final[tuple[type, ...]] = ( + create_config_class(SAMPLE_JSON_PROVIDER), + create_responses_config_class(SAMPLE_JSON_PROVIDER), +) + + +async def _resolve_access_token() -> str: + return "token" + + +@dataclass(frozen=True, slots=True) +class BuilderInputs: + constructor: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + call: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({})) + api_base_option: str | None = None + + +INPUTS_BY_BUILDER: Final[Mapping[str, BuilderInputs]] = MappingProxyType( + { + "anthropic.skills.transformation.AnthropicSkillsConfig": BuilderInputs(call={"endpoint": "skills"}), + "azure_ai.image_edit.transformation.AzureFoundryFluxImageEditConfig": BuilderInputs( + call={"litellm_params": {"api_version": "2025-04-01-preview"}} + ), + "bedrock.chat.agentcore.transformation.AmazonAgentCoreConfig": BuilderInputs( + call={"model": "agentcore/arn:aws:bedrock-agentcore:us-west-2:123456789012:runtime/test-runtime"}, + api_base_option="aws_bedrock_runtime_endpoint", + ), + "bedrock.chat.invoke_agent.transformation.AmazonInvokeAgentConfig": BuilderInputs( + call={"model": "agent/AGENT_ID/ALIAS_ID", "optional_params": {"sessionID": "session-id"}} + ), + "black_forest_labs.image_edit.transformation.BlackForestLabsImageEditConfig": BuilderInputs( + call={"model": "flux-kontext-pro"} + ), + "black_forest_labs.image_generation.transformation.BlackForestLabsImageGenerationConfig": BuilderInputs( + call={"model": "flux-pro-1.1"} + ), + "elevenlabs.text_to_speech.transformation.ElevenLabsTextToSpeechConfig": BuilderInputs( + call={"litellm_params": {ElevenLabsTextToSpeechConfig.ELEVENLABS_VOICE_ID_KEY: "voice-id"}} + ), + "meta.realtime.transformation.MetaRealtimeConfig": BuilderInputs(call={"model": MUSE_MODEL}), + "openai.evals.transformation.OpenAIEvalsConfig": BuilderInputs(call={"endpoint": "evals"}), + "openai_like.messages.transformation.JSONProviderAnthropicMessagesConfig": BuilderInputs( + constructor={"provider": SAMPLE_JSON_PROVIDER} + ), + "predibase.chat.transformation.PredibaseConfig": BuilderInputs( + call={"litellm_params": {"predibase_tenant_id": "tenant"}} + ), + "ragflow.chat.transformation.RAGFlowConfig": BuilderInputs(call={"model": "ragflow/chat/chat-id/model"}), + "scaledown.chat.transformation.ScaleDownChatConfig": BuilderInputs(call={"model": "classify"}), + "vertex_ai.audio_transcription.realtime_transformation.VertexChirpRealtimeConfig": BuilderInputs( + constructor={"resolve_access_token": _resolve_access_token, "project": "project", "location": None}, + call={"model": "chirp_3"}, + ), + "vertex_ai.interactions.transformation.VertexAIInteractionsConfig": BuilderInputs( + constructor={"mint_access_token": lambda _credentials, _project: ("token", "project")} + ), + "vertex_ai.realtime.transformation.VertexAIRealtimeConfig": BuilderInputs( + constructor={"access_token": "token", "project": "project", "location": "us-central1"} + ), + "vertex_ai.text_to_speech.transformation.VertexAILyriaTextToSpeechConfig": BuilderInputs( + call={"model": "lyria-002"} + ), + } +) + +UNCALLABLE_BUILDERS: Final[Mapping[str, str]] = MappingProxyType( + { + "base_llm.search.transformation.BaseSearchConfig": "raises-NotImplementedError-for-providers-to-override", + "bedrock.image_edit.amazon_nova_canvas_image_edit_transformation.BedrockAmazonNovaCanvasImageEditConfig": ( + "raises-because-BedrockImageEdit-builds-the-url" + ), + "sap.chat.transformation.GenAIHubOrchestrationConfig": "url-comes-from-a-live-SAP-deployment-lookup", + "sap.embed.transformation.GenAIHubEmbeddingConfig": "url-comes-from-a-live-SAP-deployment-lookup", + "triton.completion.transformation.TritonConfig": "api_base-must-be-the-full-generate-or-infer-url", + "vertex_ai.rerank.transformation.VertexAIRerankConfig": "ignores-api_base-and-needs-Vertex-credentials", + } +) + + +API_BASE_IS_THE_FULL_URL: Final[Mapping[str, str]] = MappingProxyType( + { + builder_id: "sends-the-configured-api_base-as-the-request-url" + for builder_id in ( + "base_llm.audio_transcription.transformation.BaseAudioTranscriptionConfig", + "base_llm.chat.transformation.BaseConfig", + "base_llm.completion.transformation.BaseTextCompletionConfig", + "base_llm.embedding.transformation.BaseEmbeddingConfig", + "base_llm.image_generation.transformation.BaseImageGenerationConfig", + "base_llm.image_variations.transformation.BaseImageVariationConfig", + "base_llm.vector_store.transformation.BaseDirectVectorStoreConfig", + "bedrock.realtime.transformation.BedrockRealtimeConfig", + "bedrock.search.transformation.AgentCoreSearchConfig", + "brave.search.transformation.BraveSearchConfig", + "cohere.chat.v2_transformation.CohereV2ChatConfig", + "cohere.embed.transformation.CohereEmbeddingConfig", + "dashscope.image_generation.transformation.DashScopeImageGenerationConfig", + "dataforseo.search.transformation.DataForSEOSearchConfig", + "datarobot.chat.transformation.DataRobotConfig", + "duckduckgo.search.transformation.DuckDuckGoSearchConfig", + "google_pse.search.transformation.GooglePSESearchConfig", + "searchapi.search.transformation.SearchAPIConfig", + "tinyfish.search.transformation.TinyfishSearchConfig", + "vertex_ai.text_to_speech.transformation.VertexAITextToSpeechConfig", + "vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation." + "VertexAIPartnerModelsAnthropicMessagesConfig", + ) + } +) + + +HOST_NAMED_LIKE_A_STRIPPED_SUFFIX_IS_NOT_KEPT: Final[Mapping[str, str]] = MappingProxyType( + { + "deepinfra.rerank.transformation.DeepinfraRerankConfig": "removes-every-openai-substring-on-main-too", + "minimax.videos.transformation.MinimaxVideoConfig": "strips-surface-suffixes-to-the-scheme-on-main-too", + "volcengine.videos.transformation.VolcEngineVideoConfig": "strips-surface-suffixes-to-the-scheme-on-main-too", + "vertex_ai.audio_transcription.realtime_transformation.VertexChirpRealtimeConfig": "returns-a-grpc-host", + "azure.realtime.http_transformation.AzureRealtimeHTTPConfig": "replaces-the-host-with-a-route-segment-on-main-too", + "bedrock.responses.transformation.BedrockOpenAIResponsesConfig": "replaces-the-host-with-a-route-segment-on-main-too", + "bedrock_mantle.chat.claude_transformation.BedrockMantleClaudeChatConfig": "replaces-the-host-with-a-route-segment-on-main-too", + "bedrock_mantle.messages.transformation.BedrockMantleAnthropicMessagesConfig": "replaces-the-host-with-a-route-segment-on-main-too", + "bedrock_mantle.responses.transformation.BedrockMantleResponsesAPIConfig": "replaces-the-host-with-a-route-segment-on-main-too", + "deepseek.messages.transformation.DeepSeekAnthropicMessagesConfig": "replaces-the-host-with-a-route-segment-on-main-too", + "ragflow.chat.transformation.RAGFlowConfig": "replaces-the-host-with-a-route-segment-on-main-too", + "scaledown.chat.transformation.ScaleDownChatConfig": "replaces-the-host-with-a-route-segment-on-main-too", + } +) + + +def _builder_id(cls: type) -> str: + return f"{cls.__module__.removeprefix('litellm.llms.')}.{cls.__qualname__}" + + +def _module_name(path: Path) -> str: + return ".".join(path.relative_to(PACKAGE_ROOT).with_suffix("").parts) + + +def _builder_modules() -> tuple[ModuleType, ...]: + builder_paths: Final = tuple(path for path in LLMS_ROOT.rglob("*.py") if "def get_complete_url" in path.read_text()) + return tuple(importlib.import_module(_module_name(path)) for path in sorted(builder_paths)) + + +def _defined_builders(module: ModuleType) -> tuple[type, ...]: + classes: Final = (*(cls for _, cls in inspect.getmembers(module, inspect.isclass)), *FACTORY_BUILDERS) + return tuple(cls for cls in classes if cls.__module__ == module.__name__ and "get_complete_url" in vars(cls)) + + +def _all_builders() -> tuple[type, ...]: + builders: Final = itertools.chain.from_iterable(_defined_builders(module) for module in _builder_modules()) + return tuple(sorted(builders, key=_builder_id)) + + +def _skip_reason(cls: type) -> str | None: + if getattr(vars(cls)["get_complete_url"], "__isabstractmethod__", False): + return "abstract-get_complete_url" + return UNCALLABLE_BUILDERS.get(_builder_id(cls)) + + +def _builder_param(cls: type) -> ParameterSet: + reason: Final = _skip_reason(cls) + if reason is None: + return pytest.param(cls, id=_builder_id(cls)) + return pytest.param(cls, id=f"{_builder_id(cls)}-skipped-{reason}", marks=pytest.mark.skip(reason=reason)) + + +def _unimplemented(*_args: object, **_kwargs: object) -> object: + raise NotImplementedError + + +def _instantiable(cls: type) -> type: + abstract_methods: Final[frozenset[str]] = getattr(cls, "__abstractmethods__", frozenset()) + if not abstract_methods: + return cls + return type(cls.__name__, (cls,), dict.fromkeys(abstract_methods, _unimplemented)) + + +def _call_arguments(builder: object, inputs: BuilderInputs, api_base: str) -> dict[str, object]: + parameters: Final = inspect.signature(getattr(builder, "get_complete_url")).parameters.values() + named: Final = tuple(p for p in parameters if p.kind not in (p.VAR_POSITIONAL, p.VAR_KEYWORD)) + api_base_options: Final = ( + {} if inputs.api_base_option is None else {"optional_params": {inputs.api_base_option: api_base}} + ) + values: Final = {**ARGUMENTS_BY_NAME, "api_base": api_base, **inputs.call, **api_base_options} + missing: Final = tuple(p.name for p in named if p.default is p.empty and p.name not in values) + assert not missing, f"add a value for {missing} to ARGUMENTS_BY_NAME or INPUTS_BY_BUILDER" + return {p.name: copy.deepcopy(values[p.name]) for p in named if p.name in values} + + +def _built_url(cls: type, api_base: str) -> str: + inputs: Final = INPUTS_BY_BUILDER.get(_builder_id(cls), BuilderInputs()) + builder: Final = _instantiable(cls)(**inputs.constructor) + result: Final = getattr(builder, "get_complete_url")(**_call_arguments(builder, inputs, api_base)) + return str(result[0] if isinstance(result, tuple) else result) + + +def _without_one_trailing_path_slash(url: str) -> str: + parts: Final = urlsplit(url) + return urlunsplit(parts._replace(path=parts.path.removesuffix("/"))) + + +@pytest.mark.parametrize("builder_class", [_builder_param(cls) for cls in _all_builders()]) +def test_get_complete_url_keeps_one_slash_after_a_trailing_slash_api_base(builder_class: type) -> None: + url: Final = _built_url(builder_class, API_BASE_WITH_TRAILING_SLASH) + + assert "//" not in urlsplit(url).path, url + + +@pytest.mark.parametrize("builder_class", [_builder_param(cls) for cls in _all_builders()]) +def test_get_complete_url_sends_the_same_url_with_or_without_a_trailing_slash(builder_class: type) -> None: + with_slash: Final = _built_url(builder_class, API_BASE_WITH_TRAILING_SLASH) + without_slash: Final = _built_url(builder_class, API_BASE_WITHOUT_TRAILING_SLASH) + + if _builder_id(builder_class) in API_BASE_IS_THE_FULL_URL: + assert _without_one_trailing_path_slash(with_slash) == _without_one_trailing_path_slash(without_slash) + else: + assert with_slash == without_slash + + +def _host_preserving_builder_param(cls: type) -> ParameterSet: + reason: Final = HOST_NAMED_LIKE_A_STRIPPED_SUFFIX_IS_NOT_KEPT.get(_builder_id(cls)) + if reason is None: + return _builder_param(cls) + return pytest.param(cls, id=f"{_builder_id(cls)}-skipped-{reason}", marks=pytest.mark.skip(reason=reason)) + + +@pytest.mark.parametrize("api_base", ["https://v1", "https://api/v1", "https://openai/v1/", "https://anthropic/v1"]) +@pytest.mark.parametrize("builder_class", [_host_preserving_builder_param(cls) for cls in _all_builders()]) +def test_get_complete_url_keeps_a_host_named_like_the_suffix_it_strips(builder_class: type, api_base: str) -> None: + url: Final = _built_url(builder_class, api_base) + follows_the_api_base_host: Final = ( + urlsplit(_built_url(builder_class, "https://gateway.test/v1")).netloc == "gateway.test" + ) + + assert urlsplit(url).netloc, url + assert not follows_the_api_base_host or urlsplit(url).netloc == urlsplit(api_base).netloc, url diff --git a/tests/unit/llms/test_predibase_transformation.py b/tests/unit/llms/test_predibase_transformation.py index 20d547c3b63..a02840ed4ef 100644 --- a/tests/unit/llms/test_predibase_transformation.py +++ b/tests/unit/llms/test_predibase_transformation.py @@ -1,3 +1,4 @@ +from typing import Final from unittest.mock import AsyncMock, Mock import httpx @@ -84,6 +85,19 @@ def test_predibase_get_complete_url_stream_and_non_stream(): assert stream_url.endswith("/generate_stream") +@pytest.mark.parametrize(("model", "stream"), [("generate", False), ("generate_stream", True)]) +def test_predibase_get_complete_url_appends_the_route_to_a_model_named_like_it(model: str, stream: bool) -> None: + url: Final = PredibaseConfig().get_complete_url( + api_base="https://serving.example.com", + api_key="test-key", + model=model, + optional_params={"stream": stream}, + litellm_params={"predibase_tenant_id": "tenant-123"}, + ) + + assert url.endswith(f"/llms/{model}/{model}") + + def test_predibase_get_complete_url_missing_tenant_id(): config = PredibaseConfig() diff --git a/tests/unit/llms/vertex_ai/test_vertex_llm_base.py b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py index 17a92b31284..dd2b86b6f81 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_llm_base.py +++ b/tests/unit/llms/vertex_ai/test_vertex_llm_base.py @@ -977,6 +977,22 @@ class TestVertexBase: assert result_url == f"{gateway_api_base}:embedContent" + def test_check_custom_proxy_vertex_api_base_with_path_and_trailing_slash_appends_endpoint_once(self): + gateway_api_base: Final = "https://gateway.example.com/vertex/v1/projects/p/locations/us-central1/endpoints/123" + + _, result_url = VertexBase()._check_custom_proxy( + api_base=f"{gateway_api_base}/", + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint="predict", + stream=None, + auth_header="Bearer token123", + url="https://us-central1-aiplatform.googleapis.com/v1/projects/p/locations/us-central1/endpoints/123:predict", + model="123", + ) + + assert result_url == f"{gateway_api_base}:predict" + def test_check_custom_proxy_vertex_api_base_with_version_path_grafts_default_path(self): vertex_base = VertexBase() From a8113b0dd721804c752a6f1952367fb642205224 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 08:50:19 +0000 Subject: [PATCH 059/235] fix(responses)!: round-trip Anthropic compaction blocks as compaction items (#46754) * fix(responses): round-trip Anthropic compaction blocks as compaction items /v1/responses on Anthropic routes now returns Claude's compaction block as a compaction output item (and streams it as output_item.added/done), and a compaction input item is replayed to Claude as its native compaction block, so the next turn reuses the summary instead of compacting again. Other routes receive the summary as assistant text. The Anthropic stream handler now keeps the summary text that arrives in compaction_delta. Fixes #41456 * fix(responses): replay compaction blocks natively only when the request compacts * fix(responses): carry compaction through dict-form edits, SDK replay, affinity, and guardrails Read Anthropic's dict-form context_management, accept SDK ResponseCompactionItem objects on replay, strip the encrypted_content affinity wrap before decoding, scan and rewrite compaction summaries in Responses guardrails, drop the history a summary replay replaces, send the summary as a user message so thinking ordering holds, and send the streamed compaction .done with the newest block. * fix(responses): count, route, guard, and redact compaction items consistently A compaction item now reads as its summary everywhere the proxy counts or routes a Responses request: pre-call context checks, the TPM limiter, the input token endpoint, the stream usage estimate, and the previous_response_id session rebuild all drop the history the item replaces. A Claude compaction item sent to a non-Anthropic model (including a fallback) becomes a user summary message instead of an encrypted blob the provider rejects, and Claude on Bedrock Claude Platform takes the summary path. Output guardrails see the summary of a returned compaction item, and logging redaction hides it. * fix(responses): start previous_response_id replay at a stored compaction and count reservations from it * fix(responses): keep a stored compaction summary whole in spend logs * fix(responses): keep stored compaction items summary-only, redact compaction_delta, and finish compaction before reasoning A spend-log row now stores a top-level compaction input item as its summary alone, so extra keys in a client-built item no longer reach the database whole. Message redaction also nulls the streamed compaction_delta, and the streaming bridge sends the compaction item's done event before the reasoning item that follows it. Adds the audit's integration tests for compaction items on the wire. * test(responses): cover stored compaction items, compaction_delta redaction, and compaction done order Adds integration cells for the summary-only spend row of a caller-sent compaction item and its previous_response_id replay, a redacted chat stream row that holds no streamed compaction_delta, and a compaction then thinking stream whose done events follow output order. * fix(spend_logs): honor the redaction header on failure rows for litellm_metadata routes A failed /v1/responses or /v1/messages call keeps its proxy request headers in litellm_metadata, but the failure row read redaction headers from metadata only, so x-litellm-enable-message-redaction never redacted those rows. * fix(spend_logs): lift only the redaction opt-in header into failure rows The failure hook on litellm_metadata routes copied every real request header into the caller's own metadata dict, so the stored request body gained the headers, a caller's metadata.headers was replaced, and a disable header loosened global redaction on failed calls. It now copies only the enable headers into a new litellm_params metadata dict, and only when the caller sent no metadata.headers, so no failure row is less redacted than on main * test(responses): cover failure-row redaction headers and caller metadata on litellm_metadata routes * fix(responses): keep earlier-turn instructions after compaction and count compaction items by their summary A previous_response_id rebuild that starts at a compaction now keeps the system and developer messages of every earlier turn. Token estimates for raw Responses input count a compaction item as its summary text instead of the serialized item. * fix(responses): drop the stored history a caller-sent compaction item replaces on previous_response_id * fix(responses): count only message items with a string role as compaction instructions * test(responses): flatten the last two-for comprehension in the compaction wire test * fix(responses): replay from a compaction in stored Responses output and SDK compaction items A previous_response_id whose stored row is a Responses output holding a compaction item now starts its replay at that compaction, and the handler dumps SDK compaction objects before checking for a summary. Adds tests for the uncovered compaction paths. * fix(responses): keep the stored tool call a function_call_output answers when a compaction item replaces history * fix(responses): keep a stored tool call on every turn after a compaction item and bound the stored summary A previous_response_id rebuild that starts at a turn holding a compaction item now sends again each stored call a replayed tool output answers, so a retry or a later turn no longer loses the turn-1 call. The start is picked only from rows up to the referenced response. A spend row keeps a compaction summary up to the larger of 64K chars and MAX_STRING_LENGTH_PROMPT_IN_DB instead of with no bound. * fix(responses): replay a compacted session only through the referenced response A later sibling turn that sent a compaction item put its summary between a stored tool call and its tool output, so Claude answered 400 on a turn that continued the first of two retried turns, and on a retry sent without the item * fix(spend-logs): keep main's stored value for a prompt length limit under 2 At a MAX_STRING_LENGTH_PROMPT_IN_DB of 0 or 1 the shared truncation helper stored only the truncation marker, where main stores the marker followed by the whole value * fix(spend-logs): keep main's stored tail for a negative prompt length limit At a negative MAX_STRING_LENGTH_PROMPT_IN_DB the shared truncation helper stored one more character of each value than main, since it dropped main's clamp of the end slice to the limit minus the start * fix(responses): keep a compaction item when a guardrail rewrites its summary through structured_messages * test(responses): run the compaction wire tests on the worker database under isolated workers * fix(router): keep the encrypted content affinity pin out of provider requests * fix(proxy): inspect compaction summaries in guardrails, strip a client-sent affinity pin, and price failures from the newest compaction * fix(proxy): count a failed chat or /v1/messages call on its whole history, not from a compaction-typed message * fix(proxy): trim a failed call's input at the newest compaction only when the client called /v1/responses * fix(proxy): trim a failed call's input at the newest compaction when the proxy dispatched it as a Responses call * test(proxy): run the dispatched-call-type failure test through the real request setup * fix(proxy): count a failed Cursor stream's input from the newest compaction item The Cursor route maps to no call type, so the stream generator's failure hook fell back to the route and counted the whole history. Pass the processor's route_type. * fix(proxy): count a broken Cursor stream's prompt from the newest compaction item * fix(responses): keep a tool call a later output answers when trimming before a compaction item * fix(responses): move a compaction-split tool call's turn to right before its output on both compaction routes * perf(responses): look up moved turn items in a set so a long compaction-split turn trims in linear time * test(responses): measure the compaction-split turn move as a CodSpeed benchmark instead of a clock read in unit tests * fix(responses): move only calls still open at the compaction, and bill a disconnected stream from the newest compaction item * fix(responses): restore only the newest stored call when a reused call_id is answered after a compaction item * fix(spend logs): bound a Responses compaction item's summary like a chat block instead of cutting its encrypted_content * test(proxy): price a failed dispatched call through a real ProxyLogging instead of mocked hooks * fix(responses): finish a stream whose Claude compaction failed and group answered call ids by turn once * fix(responses): count input_tokens after a compaction item the way inference sends it --- litellm/constants.py | 1 + litellm/litellm_core_utils/redact_messages.py | 57 +- litellm/llms/anthropic/chat/handler.py | 14 +- .../guardrail_translation/handler.py | 82 +- litellm/proxy/_lazy_openapi_snapshot.json | 3 + litellm/proxy/common_request_processing.py | 6 + litellm/proxy/guardrails/_content_utils.py | 17 +- .../hooks/parallel_request_limiter_v3.py | 7 +- .../proxy/hooks/proxy_track_cost_callback.py | 49 +- litellm/proxy/litellm_pre_call_utils.py | 1 + litellm/proxy/proxy_server.py | 2 + .../proxy/response_api_endpoints/endpoints.py | 21 +- litellm/proxy/spend_tracking/input_tokens.py | 11 +- .../spend_tracking/spend_tracking_utils.py | 126 +- litellm/proxy/utils.py | 31 +- .../compaction_items.py | 329 +++++ .../count_request.py | 21 +- .../handler.py | 3 + .../session_handler.py | 164 ++- .../streaming_iterator.py | 78 +- .../tool_search.py | 18 +- .../transformation.py | 265 ++++- litellm/responses/main.py | 14 +- litellm/responses/streaming_iterator.py | 1 + litellm/router.py | 1 + litellm/types/litellm_params.py | 3 + litellm/types/llms/anthropic.py | 6 + litellm/types/llms/openai.py | 2 + .../test_responses_compaction_benchmarks.py | 35 + .../test_responses_compaction_items_wire.py | 1057 +++++++++++++++++ .../test_redact_messages.py | 22 + ...test_openai_responses_guardrail_handler.py | 180 +++ .../proxy/guardrails/test_content_utils.py | 41 + .../hooks/test_proxy_track_cost_callback.py | 68 ++ tests/unit/proxy/hooks/test_tpm_concurrent.py | 50 +- .../response_api_endpoints/test_endpoints.py | 122 ++ .../proxy/spend_tracking/test_input_tokens.py | 57 + .../test_spend_tracking_utils.py | 233 +++- .../proxy/test_common_request_processing.py | 67 ++ .../unit/proxy/test_litellm_pre_call_utils.py | 2 + ..._utils_model_creation_and_error_logging.py | 135 ++- .../test_anthropic_responses_bridge.py | 639 +++++++++- .../test_compaction_items.py | 87 ++ .../test_count_request.py | 75 ++ .../test_litellm_completion_responses.py | 402 +++++++ .../test_session_handler.py | 367 +++++- .../test_streaming_iterator_transformation.py | 34 + .../test_responses_api_request_body.py | 56 + .../unit/responses/test_streaming_iterator.py | 21 + tests/unit/test_router/test_router.py | 23 + tests/unit/types/test_litellm_params.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 52 files changed, 4879 insertions(+), 230 deletions(-) create mode 100644 litellm/responses/litellm_completion_transformation/compaction_items.py create mode 100644 tests/benchmarks/test_responses_compaction_benchmarks.py create mode 100644 tests/integration/providers/test_responses_compaction_items_wire.py create mode 100644 tests/unit/responses/litellm_completion_transformation/test_compaction_items.py diff --git a/litellm/constants.py b/litellm/constants.py index 6a503463a8c..c0fe0491775 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -611,6 +611,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int( ) # timeout for adjacent services (e.g. jwt auth) MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000) MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048) +MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024 BEDROCK_MAX_POLICY_SIZE: Final = get_env_int("BEDROCK_MAX_POLICY_SIZE", 75) # One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed # identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session. diff --git a/litellm/litellm_core_utils/redact_messages.py b/litellm/litellm_core_utils/redact_messages.py index 0168cdad7c0..7f8a38cc8ec 100644 --- a/litellm/litellm_core_utils/redact_messages.py +++ b/litellm/litellm_core_utils/redact_messages.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_metadata_variable_name_from_kwargs, ) from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY +from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items from litellm.llms.vertex_ai.common_utils import ( redact_vertex_ai_metadata_from_litellm_params, redact_vertex_ai_metadata_from_logged_object, @@ -41,6 +42,8 @@ if TYPE_CHECKING: else: LiteLLMLoggingObject = Any +MESSAGE_REDACTION_ENABLE_HEADERS: Final = ("litellm-enable-message-redaction", "x-litellm-enable-message-redaction") + def _opted_out_of_message_logging(custom_logger: CustomLogger) -> bool: return hasattr(custom_logger, "message_logging") and custom_logger.message_logging is not True @@ -195,33 +198,45 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[ return { **fields, **({"reasoning_content": REDACTED_BY_LITELLM} if fields.get("reasoning_content") is not None else {}), - **{key: None for key in ("thinking_blocks", "reasoningContentBlocks", "citations") if key in fields}, + **{ + key: None + for key in ( + "thinking_blocks", + "reasoningContentBlocks", + "citations", + "compaction_blocks", + "compaction_delta", + ) + if key in fields + }, } +def _redact_field_when_present(item: object, field: str) -> None: + if getattr(item, field, None) is not None: + setattr(item, field, REDACTED_BY_LITELLM) + + def _redact_responses_api_output(output_items): """Helper to redact ResponsesAPIResponse output items.""" for output_item in output_items: + output_type = stream_item_field(output_item, "type") if getattr(output_item, "text", None) is not None: output_item.text = REDACTED_BY_LITELLM - if hasattr(output_item, "content") and isinstance(output_item.content, list): - for content_part in output_item.content: - if getattr(content_part, "text", None) is not None: - content_part.text = REDACTED_BY_LITELLM - if getattr(content_part, "refusal", None) is not None: - content_part.refusal = REDACTED_BY_LITELLM + for content_part in stream_item_items(output_item, "content"): + _redact_field_when_present(content_part, "text") + _redact_field_when_present(content_part, "refusal") - # Redact reasoning items in output array - if hasattr(output_item, "type") and output_item.type == "reasoning": - if hasattr(output_item, "summary") and isinstance(output_item.summary, list): - for summary_item in output_item.summary: - if getattr(summary_item, "text", None) is not None: - summary_item.text = REDACTED_BY_LITELLM + if output_type == "reasoning": + for summary_item in stream_item_items(output_item, "summary"): + _redact_field_when_present(summary_item, "text") - if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"): + if output_type == "compaction" and hasattr(output_item, "encrypted_content"): + output_item.encrypted_content = REDACTED_BY_LITELLM + if output_type == "function_call" and hasattr(output_item, "arguments"): output_item.arguments = REDACTED_BY_LITELLM - if hasattr(output_item, "type") and output_item.type == "custom_tool_call" and hasattr(output_item, "input"): + if output_type == "custom_tool_call" and hasattr(output_item, "input"): output_item.input = REDACTED_BY_LITELLM @@ -262,6 +277,11 @@ def _redacted_responses_api_output_item(item: object, redacted_str: str) -> obje if item_mapping.get("type") == "reasoning" and isinstance(summary, list) else {} ), + **( + {"encrypted_content": redacted_str} + if item_mapping.get("type") == "compaction" and "encrypted_content" in item_mapping + else {} + ), **( {"arguments": redacted_str} if item_mapping.get("type") == "function_call" and "arguments" in item_mapping @@ -556,13 +576,8 @@ def should_redact_message_logging(model_call_details: dict) -> bool: # User explicitly disabled redaction via header return False - possible_enable_headers: Final = [ - "litellm-enable-message-redaction", # old header. maintain backwards compatibility - "x-litellm-enable-message-redaction", # new header - ] - is_redaction_enabled_via_header = False - for header in possible_enable_headers: + for header in MESSAGE_REDACTION_ENABLE_HEADERS: if bool(request_headers.get(header, False)): is_redaction_enabled_via_header = True break diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index f0143f64be8..13ceea3e31b 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -856,11 +856,21 @@ class ModelResponseIterator: if reasoning_content is None: reasoning_content = "" elif "content" in content_block["delta"] and content_block["delta"].get("type") == "compaction_delta": - # Handle compaction delta + compaction_delta: Final = content_block["delta"]["content"] provider_specific_fields["compaction_delta"] = { "type": "compaction_delta", - "content": content_block["delta"]["content"], + "content": compaction_delta, } + if self.compaction_blocks and isinstance(compaction_delta, str): + streamed_summary: Final = self.compaction_blocks[-1].get("content") + self.compaction_blocks = [ + *self.compaction_blocks[:-1], + { + **self.compaction_blocks[-1], + "content": (streamed_summary if isinstance(streamed_summary, str) else "") + compaction_delta, + }, + ] + provider_specific_fields["compaction_blocks"] = self.compaction_blocks return ( text, diff --git a/litellm/llms/openai/responses/guardrail_translation/handler.py b/litellm/llms/openai/responses/guardrail_translation/handler.py index 84ff790ae48..3ea126c5a2a 100644 --- a/litellm/llms/openai/responses/guardrail_translation/handler.py +++ b/litellm/llms/openai/responses/guardrail_translation/handler.py @@ -65,6 +65,11 @@ from litellm.llms.base_llm.guardrail_translation.utils import ( unappliable_request_rewrite, ) from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools +from litellm.responses.litellm_completion_transformation.compaction_items import ( + compaction_summary, + decode_compaction_item, + with_compaction_summary, +) from litellm.responses.litellm_completion_transformation.custom_tools import without_malformed_tool_flags from litellm.responses.litellm_completion_transformation.tool_search import is_server_tool_search_tool from litellm.responses.litellm_completion_transformation.transformation import ( @@ -110,6 +115,16 @@ if TYPE_CHECKING: from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +_COMPACTION_SUMMARY_POSITION: Final = -1 + + +def _output_compaction_summary(item: object) -> str | None: + if stream_item_field(item, "type") != "compaction": + return None + block: Final = decode_compaction_item(stream_item_field(item, "encrypted_content")) + return None if block is None else block.summary + + class _ToolCallShape(NamedTuple): name: str | None arguments: str @@ -266,7 +281,17 @@ def _item_rewrite_field(item: Mapping[str, object]) -> str | None: return _PATCHABLE_ITEM_FIELDS.get(item_type) +def _rewritten_compaction_item(item: Mapping[str, object], rewritten: object) -> Mapping[str, object] | None: + encrypted_content: Final = item.get("encrypted_content") + summary: Final = rewritten.get("content") if isinstance(rewritten, Mapping) else None + if not isinstance(encrypted_content, str) or not isinstance(summary, str) or not summary: + return None + return {**item, "encrypted_content": with_compaction_summary(encrypted_content, summary)} + + def _rewritten_input_item(item: Mapping[str, object], rewritten: object) -> Mapping[str, object] | None: + if compaction_summary(item) is not None: + return _rewritten_compaction_item(item, rewritten) field: Final = _item_rewrite_field(item) if field is None or not isinstance(rewritten, Mapping): return None @@ -804,6 +829,11 @@ class OpenAIResponsesHandler(BaseTranslation): Override this method to customize text/image extraction logic. """ + summary: Final = compaction_summary(message) + if summary is not None: + texts_to_check.append(summary) + task_mappings.append((msg_idx, None)) + return content: Final = message.get("content", None) if content is None: return @@ -849,6 +879,12 @@ class OpenAIResponsesHandler(BaseTranslation): msg_idx = cast(int, mapping[0]) content_idx_optional = cast(int | None, mapping[1]) + if compaction_summary(messages[msg_idx]) is not None: + messages[msg_idx]["encrypted_content"] = with_compaction_summary( # rebind-ok: in-place input rewrite + messages[msg_idx]["encrypted_content"], guardrail_response + ) + continue + content = messages[msg_idx].get("content", None) if content is None: continue @@ -1116,13 +1152,17 @@ class OpenAIResponsesHandler(BaseTranslation): # delivered, so a truncated buffer still reports text undeliverable. # # ------------------------------------------------------------------ # if final_chunk.get("type") == "response.output_item.done": + finished_summary: Final = _output_compaction_summary(stream_item_field(final_chunk, "item")) model_response_stream: Final = ( OpenAiResponsesToChatCompletionStreamIterator.translate_responses_chunk_to_openai_stream(final_chunk) ) tool_calls: Final = model_response_stream.choices[0].delta.tool_calls - if tool_calls: + if tool_calls or finished_summary is not None: inputs = GenericGuardrailAPIInputs() - inputs["tool_calls"] = cast(list[ChatCompletionToolCallChunk], tool_calls) + if finished_summary is not None: + inputs["texts"] = [finished_summary] + if tool_calls: + inputs["tool_calls"] = cast(list[ChatCompletionToolCallChunk], tool_calls) if hasattr(model_response_stream, "model") and model_response_stream.model: inputs["model"] = model_response_stream.model await guardrail_to_apply.apply_guardrail( @@ -1214,6 +1254,14 @@ class OpenAIResponsesHandler(BaseTranslation): else: setattr(event, field, value) + @staticmethod + def _write_compaction_summary(item: object, summary: str) -> None: + encrypted_content: Final = stream_item_field(item, "encrypted_content") + if isinstance(encrypted_content, str): + OpenAIResponsesHandler._write_event_field( + item, "encrypted_content", with_compaction_summary(encrypted_content, summary) + ) + def _sync_stream_events_with_rewrites( self, stream_events: Sequence[Any], @@ -1256,11 +1304,13 @@ class OpenAIResponsesHandler(BaseTranslation): output_index: int, rewrites_by_position: Mapping[tuple[int, int], str], ) -> None: - content: Final = item.get("content") if isinstance(item, dict) else getattr(item, "content", None) - if not isinstance(content, list): + summary_rewrite: Final = rewrites_by_position.get((output_index, _COMPACTION_SUMMARY_POSITION)) + if summary_rewrite is not None: + OpenAIResponsesHandler._write_compaction_summary(item, summary_rewrite) return + content: Final = stream_item_items(item, "content") for (item_idx, content_idx), rewritten in rewrites_by_position.items(): - if item_idx != output_index or content_idx >= len(content): + if item_idx != output_index or not 0 <= content_idx < len(content): continue OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten) @@ -1481,13 +1531,13 @@ class OpenAIResponsesHandler(BaseTranslation): message_items: Final = tuple( item for item in output_items if stream_item_field(item, "type") not in _OUTPUT_TOOL_CALL_ITEM_TYPES ) + message_parts: Final = chain.from_iterable(stream_item_items(item, "content") for item in message_items) + part_texts: Final = tuple( + text for part in message_parts if isinstance(text := stream_item_field(part, "text"), str) and text + ) + summaries: Final = tuple(summary for item in output_items if (summary := _output_compaction_summary(item))) return StreamingScanKey( - texts=tuple( - text - for item in message_items - for part in stream_item_items(item, "content") - if isinstance(text := stream_item_field(part, "text"), str) and text - ), + texts=(*part_texts, *summaries), tool_calls=tuple( stream_item_fingerprint(item) for item in output_items @@ -1610,6 +1660,12 @@ class OpenAIResponsesHandler(BaseTranslation): ) return + summary: Final = _output_compaction_summary(output_item) + if summary is not None: + texts_to_check.append(summary) + task_mappings.append((output_idx, _COMPACTION_SUMMARY_POSITION)) + return + # Handle both GenericResponseOutputItem and dict content: list[OutputText] | list[dict] | None = None if isinstance(output_item, BaseModel): @@ -1677,6 +1733,10 @@ class OpenAIResponsesHandler(BaseTranslation): output_item = response_output[output_idx] + if content_idx == _COMPACTION_SUMMARY_POSITION: + self._write_compaction_summary(output_item, guardrail_response) + continue + # Handle both GenericResponseOutputItem, BaseModel, and dict if isinstance(output_item, GenericResponseOutputItem): if output_item.content and content_idx < len(output_item.content): diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 545b317055a..4572e59944b 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -28240,6 +28240,9 @@ }, { "$ref": "#/components/schemas/OutputToolSearchCall" + }, + { + "$ref": "#/components/schemas/ResponseCompactionItem" } ] }, diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 6fda5780f63..23faa964d5f 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -1985,6 +1985,7 @@ class ProxyBaseLLMRequestProcessing: self.data = data self.served_model = served_model self._tags_before_guardrails: frozenset[str] | None = None + self.route_type: str | None = None @property def litellm_call_id(self) -> str | None: @@ -2434,6 +2435,7 @@ class ProxyBaseLLMRequestProcessing: ## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call ## IMPORTANT Note: - initialize this before running pre-call checks. Ensures we log rejected requests to langfuse. + self.route_type = route_type logging_obj, self.data = litellm.utils.function_setup( original_function=route_type, rules_obj=litellm.utils.Rules(), @@ -2800,6 +2802,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, original_exception=exc, request_data=self.data, + dispatched_call_type=self.route_type, ) return await open_sse_before_first_byte( @@ -2864,6 +2867,7 @@ class ProxyBaseLLMRequestProcessing: "skip_pre_call_logic=True requires litellm_logging_obj to be set in data. " "Ensure common_processing_pre_call_logic was called before using this parameter." ) + self.route_type = route_type else: self.data, logging_obj = await self._pre_call_with_fallbacks( request=request, @@ -3473,6 +3477,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, original_exception=exc, request_data=request_data, + dispatched_call_type=self.route_type, ) except HTTPException: return @@ -4142,6 +4147,7 @@ class ProxyBaseLLMRequestProcessing: user_api_key_dict=user_api_key_dict, original_exception=e, request_data=self.data, + dispatched_call_type=self.route_type, ) # Use transformed exception if callback returned one, otherwise use original if transformed_exception is not None: diff --git a/litellm/proxy/guardrails/_content_utils.py b/litellm/proxy/guardrails/_content_utils.py index 64c935ce0d0..17bea031794 100644 --- a/litellm/proxy/guardrails/_content_utils.py +++ b/litellm/proxy/guardrails/_content_utils.py @@ -11,6 +11,12 @@ every text fragment. from collections.abc import Callable, Iterator, Mapping, Sequence from typing import Any, Final +from litellm.responses.litellm_completion_transformation.compaction_items import ( + compaction_summary, + input_items, + with_visited_compaction_summary, +) + # Call types whose body carries free-form chat / prompt text that # text-content guardrails (banned keywords, content moderation, secret # detection, …) should inspect. The proxy ingress passes ``route_type`` @@ -95,12 +101,12 @@ def _coerce_input_to_messages(input_value: object) -> list[dict[str, object]]: """Coerce a Responses-API ``data["input"]`` value into chat-style messages.""" if isinstance(input_value, str): return [{"role": "user", "content": input_value}] - if not isinstance(input_value, list): - return [] messages: Final[list[dict[str, object]]] = [] - for item in input_value: + for item in input_items(input_value): if isinstance(item, str): messages.append({"role": "user", "content": item}) + elif (summary := compaction_summary(item)) is not None: + messages.append({"role": "user", "content": summary}) elif isinstance(item, dict): if _part_text(item) is not None: messages.append({"role": item.get("role") or "user", "content": [item]}) @@ -189,11 +195,14 @@ def walk_user_text(data: dict[str, object], visit: Callable[[str], str]) -> int: data["input"] = visit(input_value) return visited if isinstance(input_value, list): - for idx, item in enumerate(input_value): + for idx, item in enumerate(input_items(input_value)): if isinstance(item, str): if item: visited += 1 input_value[idx] = visit(item) + elif (compaction := with_visited_compaction_summary(item, visit)) is not None: + visited += 1 + input_value[idx] = compaction elif isinstance(item, dict): if _part_text(item) is not None: visited += 1 diff --git a/litellm/proxy/hooks/parallel_request_limiter_v3.py b/litellm/proxy/hooks/parallel_request_limiter_v3.py index a8ba8d65785..a2bb691b401 100644 --- a/litellm/proxy/hooks/parallel_request_limiter_v3.py +++ b/litellm/proxy/hooks/parallel_request_limiter_v3.py @@ -73,6 +73,7 @@ from litellm.proxy.hooks.batch_enqueued_tokens import ( canonical_provider_batch_id, ) from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit +from litellm.responses.litellm_completion_transformation.compaction_items import billable_input_since_newest_compaction from litellm.router_utils.add_retry_fallback_headers import ( ensure_response_additional_headers, response_has_hidden_params, @@ -1247,7 +1248,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): estimable_data.get("input"), ) ) - messages, prompt, input_text = selected_fields + messages, prompt, raw_input = selected_fields + input_text: Final = ( + billable_input_since_newest_compaction(raw_input) if call_type in RESPONSES_API_CALL_TYPES else raw_input + ) total_chars: Final = ( len(get_str_from_messages(messages)) @@ -3835,6 +3839,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger): return LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=data.get("input") or "", responses_api_request=data, + only_since_compaction=True, ) @staticmethod diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 994310c6619..6801c8521e5 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -22,6 +22,7 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost +from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.auth.auth_checks import ( @@ -127,6 +128,9 @@ _CAPTURED_IDENTITY_CALL_TYPES: Final[frozenset[str]] = frozenset( ) +_OBJECT_MAPPING: Final = TypeAdapter(ObjectMapping) + + def _proxy_stamped_used_client_oauth_token( request_data: Mapping[str, object], request_route: str | None ) -> bool | None: @@ -183,6 +187,26 @@ def _failure_snapshot_with_guardrails_recorded_after_it( return {**failure_snapshot, "guardrail_information": recorded_guardrail_information} +def _guardrail_info_lifted_from_litellm_metadata( + litellm_metadata: ObjectMapping, *, guardrail_info_already_set: bool +) -> ObjectMapping: + guardrail_info: Final = litellm_metadata.get("standard_logging_guardrail_information") + if guardrail_info is None or guardrail_info_already_set: + return {} + return {"standard_logging_guardrail_information": guardrail_info} + + +def _redaction_opt_in_lifted_from_litellm_metadata( + litellm_metadata: ObjectMapping, *, caller_set_headers: bool +) -> ObjectMapping: + request_headers: Final = litellm_metadata.get("headers") + if caller_set_headers or not isinstance(request_headers, dict): + return {} + headers: Final = _OBJECT_MAPPING.validate_python(request_headers) + opt_in_headers: Final = {name: headers[name] for name in MESSAGE_REDACTION_ENABLE_HEADERS if name in headers} + return {"headers": opt_in_headers} if opt_in_headers else {} + + def _proxy_spend_writer() -> DBSpendUpdateWriter: from litellm.proxy.proxy_server import proxy_logging_obj @@ -417,13 +441,17 @@ class _ProxyDBLogger(CustomLogger): existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata}) litellm_metadata_bucket: Final = request_data.get("litellm_metadata") - if ( - isinstance(litellm_metadata_bucket, dict) - and "standard_logging_guardrail_information" not in existing_metadata - ): - guardrail_info: Final = litellm_metadata_bucket.get("standard_logging_guardrail_information") - if guardrail_info is not None: - existing_metadata["standard_logging_guardrail_information"] = guardrail_info + litellm_metadata: Final[ObjectMapping] = ( + _OBJECT_MAPPING.validate_python(litellm_metadata_bucket) + if isinstance(litellm_metadata_bucket, dict) + else {} + ) + existing_metadata.update( + _guardrail_info_lifted_from_litellm_metadata( + litellm_metadata, + guardrail_info_already_set="standard_logging_guardrail_information" in existing_metadata, + ) + ) if "litellm_params" not in request_data: request_data["litellm_params"] = {} @@ -438,7 +466,12 @@ class _ProxyDBLogger(CustomLogger): request_data["litellm_params"]["proxy_server_request"] = ( request_data.get("proxy_server_request") or existing_litellm_params.get("proxy_server_request") or {} ) - request_data["litellm_params"]["metadata"] = existing_metadata + request_data["litellm_params"]["metadata"] = { + **existing_metadata, + **_redaction_opt_in_lifted_from_litellm_metadata( + litellm_metadata, caller_set_headers="headers" in existing_metadata + ), + } # Preserve model name and custom_llm_provider if "model" not in request_data: diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d34787cd5e9..8a45f658f36 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -353,6 +353,7 @@ LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset( _UNTRUSTED_ROOT_CONTROL_FIELDS: Final = ( "weights", "_router_weights", + "_encrypted_content_affinity_pinned", "fallback_depth", "_target_order", "attempted_targets", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b48b5ab1c1e..f632317d9a8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10384,6 +10384,7 @@ async def async_data_generator( request: Request | None = None, *, responses_stream_errors: bool = False, + dispatched_call_type: str | None = None, ): verbose_proxy_logger.debug("inside generator") stream_completed = False @@ -10599,6 +10600,7 @@ async def async_data_generator( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=request_data, + dispatched_call_type=dispatched_call_type, ) client_error: Final[Exception] = e if replacement is None else replacement verbose_proxy_logger.debug( diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index c918ea3f1d1..312c5124ca5 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -37,6 +37,7 @@ from litellm.constants import EMPTY_MAPPING, MANAGED_BACKGROUND_RESPONSE_METADAT from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.litellm_core_utils.core_helpers import as_str_mapping, get_metadata_variable_name_from_kwargs from litellm.litellm_core_utils.internal_call_metadata import is_background_response, requests_background +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.tool_fields import TOOL_FIELDS, first_validation_error from litellm.llms.openai.responses.guardrail_translation.handler import ( @@ -79,7 +80,8 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.batch_attributio request_tags_from_request_data, ) from litellm.proxy.route_llm_request import raise_if_required_body_param_missing -from litellm.proxy.utils import handle_exception_on_proxy +from litellm.proxy.utils import count_request_input_tokens, handle_exception_on_proxy +from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.responses.litellm_completion_transformation.count_request import MODEL_FILE_ID_MAPPING_FIELD from litellm.responses.streaming_iterator import GuardedResponsesWsFrame, ResponsesWsFrameGuard, spawn_logging_task from litellm.router import Router @@ -94,7 +96,6 @@ from litellm.types.token_count import TokenCountSourceFields from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, TokenCountResponse if TYPE_CHECKING: - from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.utils import ProxyLogging router: Final = APIRouter() @@ -532,6 +533,14 @@ async def cursor_model_list( return await model_list(user_api_key_dict=user_api_key_dict) +def _dispatched_prompt_token_counter(logging_obj: LiteLLMLoggingObj) -> Callable[[], int]: + return lambda: count_request_input_tokens( + model=str(logging_obj.model_call_details.get("model") or ""), + request_input=messages_since_newest_compaction(logging_obj.model_call_details.get("messages")), + system_input=None, + ) + + @router.post( "/cursor/chat/completions", dependencies=[ @@ -656,6 +665,11 @@ async def cursor_chat_completions( model=request_data.get("model", ""), custom_llm_provider=None, logging_obj=logging_obj, + count_prompt_tokens=( + _dispatched_prompt_token_counter(logging_obj) + if isinstance(logging_obj, LiteLLMLoggingObj) + else None + ), ) # Use async_data_generator to format as SSE return async_data_generator( @@ -663,6 +677,7 @@ async def cursor_chat_completions( user_api_key_dict=user_api_key_dict, request_data=request_data, request=request, + dispatched_call_type=processor.route_type, ) # Otherwise, use the default generator return async_data_generator( @@ -670,6 +685,7 @@ async def cursor_chat_completions( user_api_key_dict=user_api_key_dict, request_data=request_data, request=request, + dispatched_call_type=processor.route_type, ) try: @@ -1585,6 +1601,7 @@ def _responses_input_as_token_count_messages( transformed: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=input_value, responses_api_request=request_params, + only_since_compaction=True, ) return tuple( message if isinstance(message, dict) else message.model_dump(exclude_none=True) for message in transformed diff --git a/litellm/proxy/spend_tracking/input_tokens.py b/litellm/proxy/spend_tracking/input_tokens.py index 6c7083fb6db..927152296e9 100644 --- a/litellm/proxy/spend_tracking/input_tokens.py +++ b/litellm/proxy/spend_tracking/input_tokens.py @@ -19,6 +19,10 @@ from typing import Final import litellm from litellm._logging import verbose_proxy_logger +from litellm.responses.litellm_completion_transformation.compaction_items import ( + billable_input_since_newest_compaction, + holds_compaction_summary, +) from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext from litellm.rust_bridge.token_counter import ( @@ -54,10 +58,11 @@ async def count_input_tokens( (model, rust_tokenizer(model)) for model in models ) groups: Final[tuple[RustTokenizer | None, ...]] = tuple(dict.fromkeys(tokenizer for _, tokenizer in tokenizers)) + counted_raw_body: Final = None if holds_compaction_summary(request_body.get("input")) else raw_body group_counts: Final = [ await _count_group( request_body=request_body, - raw_body=raw_body, + raw_body=counted_raw_body, tokenizer=tokenizer, models=tuple(model for model, selected in tokenizers if selected == tokenizer), ) @@ -140,7 +145,9 @@ def count_input_tokens_for_model(request_body: dict, model: str) -> int | None: if "prompt" in request_body: return _count_text_tokens(model=model, text=request_body.get("prompt")) if "input" in request_body: - return _count_text_tokens(model=model, text=request_body.get("input")) + return _count_text_tokens( + model=model, text=billable_input_since_newest_compaction(request_body.get("input")) + ) if "query" in request_body or "documents" in request_body: query_tokens: Final = _count_text_tokens(model=model, text=request_body.get("query")) document_tokens: Final = _count_text_tokens( diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 62aee788ed0..94b079163e9 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -27,6 +27,7 @@ from litellm.constants import ( LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, + MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB, MCP_SPEND_LOG_MODEL_PREFIX, NON_INFERENCE_CALL_TYPES, REDACTED_BY_LITELM_STRING, @@ -70,6 +71,11 @@ from litellm.proxy._types import ( from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.proxy.utils import PrismaClient, hash_token +from litellm.responses.litellm_completion_transformation.compaction_items import ( + compaction_summary, + summary_only_compaction_content, + with_compaction_summary, +) from litellm.types.passthrough_endpoints.pass_through_endpoints import ( SETTLES_BACKGROUND_RESPONSE_KWARG, BackgroundResponseSettlement, @@ -1227,11 +1233,42 @@ def _is_spend_log_content(parent: Mapping[str, object], key: str) -> bool: ) +def _truncated_for_db(value: str, max_length: int) -> str: + if len(value) <= max_length: + return value + start_chars: Final = int(max_length * 0.35) + end_chars: Final = int(max_length * 0.65) + return ( + f"{value[:start_chars]}" + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {len(value) - start_chars - end_chars} chars. " + f"{LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." + f"{value[-min(end_chars, max_length - start_chars) :]}" + ) + + +def _compaction_summary_for_db(summary: str) -> str: + return _truncated_for_db( + summary, max(_get_max_string_length_prompt_in_db(), MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB) + ) + + +def _bounded_compaction_summary( + parent: Mapping[str, object], key: str, value: object, keep_compaction_blocks: bool +) -> str | None: + if not keep_compaction_blocks or not isinstance(value, str) or parent.get("type") != "compaction": + return None + if key == "content": + return _compaction_summary_for_db(value) + summary: Final = compaction_summary(parent) if key == "encrypted_content" else None + return None if summary is None else with_compaction_summary(value, _compaction_summary_for_db(summary)) + + def _sanitize_request_body_for_spend_logs_payload( request_body: Mapping[str, object], visited: set | None = None, max_string_length_prompt_in_db: int | None = None, mask_credentials: bool = True, + keep_compaction_blocks: bool = False, ) -> dict: """ Recursively sanitize request body to prevent logging large base64 strings or other large values. @@ -1242,11 +1279,6 @@ def _sanitize_request_body_for_spend_logs_payload( SensitiveDataMasker classifies as credentials with REDACTED_BY_LITELM_STRING, except inside tool payloads and logprobs. """ - from litellm.constants import ( - LITELLM_TRUNCATED_PAYLOAD_FIELD, - LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, - ) - if visited is None: visited = set() if max_string_length_prompt_in_db is None: @@ -1261,51 +1293,23 @@ def _sanitize_request_body_for_spend_logs_payload( def _sanitize_value(value: object, mask_credentials: bool) -> object: if isinstance(value, Mapping): return _sanitize_request_body_for_spend_logs_payload( - value, visited, max_string_length_prompt_in_db, mask_credentials + value, visited, max_string_length_prompt_in_db, mask_credentials, keep_compaction_blocks ) elif isinstance(value, list): return [_sanitize_value(item, mask_credentials) for item in value] elif isinstance(value, str): - if len(value) > max_string_length_prompt_in_db: - # Keep 35% from beginning and 65% from end (end is usually more important) - # This split ensures we keep more context from the end of conversations - start_ratio: Final = 0.35 - end_ratio: Final = 0.65 - - # Calculate character distribution - start_chars: Final = int(max_string_length_prompt_in_db * start_ratio) - end_chars = int(max_string_length_prompt_in_db * end_ratio) - - # Ensure we don't exceed the total limit - total_keep: Final = start_chars + end_chars - if total_keep > max_string_length_prompt_in_db: - end_chars = max_string_length_prompt_in_db - start_chars - - # If the string length is less than what we want to keep, just truncate normally - if len(value) <= max_string_length_prompt_in_db: - return value - - # Calculate how many characters are being skipped - skipped_chars: Final = len(value) - total_keep - - # Build the truncated string: beginning + truncation marker + end - truncated_value: Final = ( - f"{value[:start_chars]}" - f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. " - f"{LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." - f"{value[-end_chars:]}" - ) - return truncated_value - return value + return _truncated_for_db(value, max_string_length_prompt_in_db) return value - return { - k: REDACTED_BY_LITELM_STRING - if mask_credentials and _is_request_body_credential(k, v) - else _sanitize_value(v, mask_credentials and not _is_spend_log_content(request_body, k)) - for k, v in request_body.items() - if k not in _SENSITIVE_REQUEST_BODY_KEYS - } + def _stored_value(key: str, value: object) -> object: + if mask_credentials and _is_request_body_credential(key, value): + return REDACTED_BY_LITELM_STRING + bounded_summary: Final = _bounded_compaction_summary(request_body, key, value, keep_compaction_blocks) + if bounded_summary is not None: + return bounded_summary + return _sanitize_value(value, mask_credentials and not _is_spend_log_content(request_body, key)) + + return {k: _stored_value(k, v) for k, v in request_body.items() if k not in _SENSITIVE_REQUEST_BODY_KEYS} # Quoted-key form: ``"input"`` / ``'messages'`` / ``"prompt"`` followed by @@ -1820,6 +1824,33 @@ def _stored_request_without_requester_ip_address(request_body: Mapping[object, o } +def _with_bounded_compaction_summary(item: object, stored_item: object) -> object: + summary: Final = compaction_summary(item) + if summary is None or not isinstance(stored_item, Mapping): + return stored_item + return { + **_OBJECT_MAPPING.validate_python(stored_item), + "encrypted_content": summary_only_compaction_content(_compaction_summary_for_db(summary)), + } + + +def _sanitize_proxy_server_request_body(request_body: Mapping[str, object]) -> Mapping[object, object]: + sanitized: Final = _OBJECT_MAPPING.validate_python(_sanitize_request_body_for_spend_logs_payload(request_body)) + items: Final = request_body.get("input") + stored_items: Final = sanitized.get("input") + if not isinstance(items, list) or not isinstance(stored_items, list): + return sanitized + return { + **sanitized, + "input": [ + _with_bounded_compaction_summary(item, stored_item) + for item, stored_item in zip( + _OBJECT_LIST.validate_python(items), _OBJECT_LIST.validate_python(stored_items) + ) + ], + } + + def _get_proxy_server_request_for_spend_logs_payload( metadata: dict, litellm_params: Mapping[str, object], @@ -1864,8 +1895,7 @@ def _get_proxy_server_request_for_spend_logs_payload( _request_body = _convert_mapping_to_json_serializable(without_classifier_audit(_request_body)) perform_redaction(model_call_details=_request_body, result=None) - _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) - stored_request_body: Final = _OBJECT_MAPPING.validate_python(_request_body) + stored_request_body: Final = _sanitize_proxy_server_request_body(_request_body) _request_body_json_str: Final = safe_dumps( _stored_request_without_requester_ip_address(stored_request_body) if drop_requester_ip_address @@ -1939,7 +1969,9 @@ def _get_response_for_spend_logs_payload( response_obj = _convert_to_json_serializable_dict(response_obj) response_obj = perform_redaction(model_call_details={}, result=response_obj) - sanitized_wrapper: Final = _sanitize_request_body_for_spend_logs_payload({"response": response_obj}) + sanitized_wrapper: Final = _sanitize_request_body_for_spend_logs_payload( + {"response": response_obj}, keep_compaction_blocks=True + ) sanitized_response: Final = sanitized_wrapper.get("response", response_obj) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2a328971817..52b3866d04d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -89,6 +89,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( with_litellm_call_id, ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error +from litellm.responses.litellm_completion_transformation.compaction_items import messages_since_newest_compaction from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.model_listing import ModelInfoResponse from litellm.types.utils import MCP_GUARDRAIL_CALL_TYPES, CallTypes, CallTypesLiteral, ModelInfo, Usage @@ -937,7 +938,7 @@ def _system_prompt_text(system_input: object) -> str: return "".join(_prompt_block_text(block) for block in system_input) -def _count_request_input_tokens(model: str, request_input: object, system_input: object) -> int: +def count_request_input_tokens(model: str, request_input: object, system_input: object) -> int: system_text: Final = _system_prompt_text(system_input) system_tokens: Final = litellm.token_counter(model=model, text=system_text) if system_text else 0 if isinstance(request_input, str): @@ -958,7 +959,7 @@ def _estimate_dispatched_failure_usage(model: str, request_input: object, system the same tokenizer fallback interrupted streams use, so the spend log's failure row records what was sent instead of zero.""" try: - input_tokens: Final = _count_request_input_tokens( + input_tokens: Final = count_request_input_tokens( model=model, request_input=request_input, system_input=system_input ) except Exception: @@ -997,6 +998,8 @@ _INPUT_ESTIMABLE_CALL_TYPES: Final = frozenset( ) +_RESPONSES_CALL_TYPES: Final = frozenset((CallTypes.responses.value, CallTypes.aresponses.value)) + _FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[tuple[Usage, float], ...]) @@ -1022,6 +1025,7 @@ def _current_failure_usage_to_lift( provider_billed: bool = False, deployment_id: str | None = None, base_model: str | None = None, + dispatched_call_type: str | None = None, ) -> tuple[object, object] | None: """A stream that broke mid-flight still billed the provider for the chunks already delivered; the streaming handler stashes that recovered usage and @@ -1045,7 +1049,7 @@ def _current_failure_usage_to_lift( # The proxy-only error handler stamps this flag on a client disconnect too, even though the # request had already reached the provider, so a billed failure keeps its estimate. return None - estimated_usage: Final = _estimated_prompt_usage(model_call_details, request_body) + estimated_usage: Final = _estimated_prompt_usage(model_call_details, request_body, dispatched_call_type) if estimated_usage is None: return None if not provider_billed: @@ -1056,13 +1060,14 @@ def _current_failure_usage_to_lift( def _estimated_prompt_usage( - model_call_details: Mapping[str, object], request_body: Mapping[str, object] + model_call_details: Mapping[str, object], request_body: Mapping[str, object], dispatched_call_type: str | None ) -> Usage | None: """The input-only Usage estimate for the prompt this request sent, or None when the call type has no estimable input. The raw request body backfills the system prompt when the SDK bridges an endpoint (e.g. /v1/messages on a chat-completions provider) without filling optional_params.""" if str(model_call_details.get("call_type")) not in _INPUT_ESTIMABLE_CALL_TYPES: return None + dispatched_input: Final = model_call_details.get("messages") optional_params: Final = model_call_details.get("optional_params") dispatched_system: Final = ( (optional_params.get("system") or optional_params.get("instructions")) @@ -1079,7 +1084,11 @@ def _estimated_prompt_usage( ) return _estimate_dispatched_failure_usage( model=str(model_call_details.get("model") or ""), - request_input=model_call_details.get("messages"), + request_input=( + messages_since_newest_compaction(dispatched_input) + if dispatched_call_type in _RESPONSES_CALL_TYPES + else dispatched_input + ), system_input=system_input, ) @@ -1444,7 +1453,9 @@ def _provider_billed_the_failure(exception: Exception) -> bool: return _timed_out_reading_from_provider(exception) or _client_disconnected(exception) -def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: bool = False) -> Mapping[str, object]: +def _failure_fields_to_lift( + request_data: Mapping[str, object], provider_billed: bool = False, dispatched_call_type: str | None = None +) -> Mapping[str, object]: """Failure-path callbacks run after ``litellm_logging_obj`` is popped from request_data (it is not serialisable), so the caller merges these fields onto request_data first: the request start and first-handoff instants for @@ -1468,9 +1479,10 @@ def _failure_fields_to_lift(request_data: Mapping[str, object], provider_billed: provider_billed=_billed, deployment_id=_deployment_id, base_model=_base_model, + dispatched_call_type=dispatched_call_type, ) _prompt_estimate: Final = ( - _estimated_prompt_usage(_model_call_details, request_data) + _estimated_prompt_usage(_model_call_details, request_data, dispatched_call_type) if _billed and _first_handoff is not None and not _model_call_details.get("failed_attempts_usage") else None ) @@ -3634,6 +3646,7 @@ class ProxyLogging: error_type: ProxyErrorTypes | None = None, route: str | None = None, traceback_str: str | None = None, + dispatched_call_type: str | None = None, ) -> HTTPException | None: """ Allows users to raise custom exceptions/log when a call fails, without having to deal with parsing Request body. @@ -3704,7 +3717,9 @@ class ProxyLogging: request_data.update( await offload_token_count(_failure_fields_to_lift)( - request_data, provider_billed=_provider_billed_the_failure(original_exception) + request_data, + provider_billed=_provider_billed_the_failure(original_exception), + dispatched_call_type=dispatched_call_type or _call_type_for_route(user_api_key_dict.request_route), ) ) diff --git a/litellm/responses/litellm_completion_transformation/compaction_items.py b/litellm/responses/litellm_completion_transformation/compaction_items.py new file mode 100644 index 00000000000..8bd82c49090 --- /dev/null +++ b/litellm/responses/litellm_completion_transformation/compaction_items.py @@ -0,0 +1,329 @@ +import json +import uuid +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain, groupby +from types import MappingProxyType +from typing import Final, TypeVar + +from openai.types.responses.easy_input_message_param import EasyInputMessageParam +from openai.types.responses.response_compaction_item import ResponseCompactionItem +from openai.types.responses.response_create_params import ResponseInputParam +from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError + +from litellm.responses.utils import ResponsesAPIRequestUtils + +from .tool_search import chat_config_builds_anthropic_request, provider_chat_config + +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_OBJECT_LIST: Final = TypeAdapter(list[object]) +_Item: Final = TypeVar("_Item") +_INSTRUCTION_ROLES: Final = frozenset(("system", "developer")) + + +def _json_object(value: object) -> Mapping[str, object] | None: + try: + return _JSON_OBJECT.validate_python(value) + except ValidationError: + return None + + +@dataclass(frozen=True, slots=True) +class CompactionBlock: + summary: str + wire_block: Mapping[str, object] + + +def _compaction_block(value: object) -> CompactionBlock | None: + block: Final = _json_object(value) + if block is None: + return None + summary: Final = block.get("content") + if block.get("type") != "compaction" or not isinstance(summary, str) or not summary: + return None + return CompactionBlock(summary=summary, wire_block=block) + + +def _newest_compaction_block(compaction_blocks: object) -> CompactionBlock | None: + if not isinstance(compaction_blocks, list): + return None + blocks: Final = _OBJECT_LIST.validate_python(compaction_blocks) + return next((block for value in reversed(blocks) if (block := _compaction_block(value)) is not None), None) + + +def _provider_compaction_block(provider_specific_fields: object) -> CompactionBlock | None: + fields: Final = _json_object(provider_specific_fields) + return None if fields is None else _newest_compaction_block(fields.get("compaction_blocks")) + + +def provider_compaction_summary(provider_specific_fields: object) -> str | None: + block: Final = _provider_compaction_block(provider_specific_fields) + return None if block is None else block.summary + + +class _AssistantTurnFields(BaseModel): + model_config = ConfigDict(frozen=True, from_attributes=True) + + content: object = None + tool_calls: object = None + provider_specific_fields: object = None + + +def carries_only_compaction(message: object) -> bool: + fields: Final = _AssistantTurnFields.model_validate(message) + return ( + not fields.content + and not fields.tool_calls + and provider_compaction_summary(fields.provider_specific_fields) is not None + ) + + +def _choice_provider_specific_fields(choice: object) -> object: + choice_fields: Final = _json_object(choice) + message: Final = None if choice_fields is None else _json_object(choice_fields.get("message")) + return None if message is None else message.get("provider_specific_fields") + + +def response_holds_compaction_block(response: object) -> bool: + fields: Final = _json_object(response) + if fields is None: + return False + if fields.get("object") == "response": + return holds_compaction_summary(fields.get("output")) + try: + choices: Final = _OBJECT_LIST.validate_python(fields.get("choices")) + except ValidationError: + return False + return any(provider_compaction_summary(_choice_provider_specific_fields(choice)) is not None for choice in choices) + + +def compaction_output_item(provider_specific_fields: object) -> ResponseCompactionItem | None: + block: Final = _provider_compaction_block(provider_specific_fields) + if block is None: + return None + return ResponseCompactionItem( + id=f"cmp_{uuid.uuid4()}", + encrypted_content=json.dumps(dict(block.wire_block), separators=(",", ":")), + type="compaction", + ) + + +def _entry_type(entry: object) -> object: + fields: Final = _json_object(entry) + return None if fields is None else fields.get("type") + + +def _has_entry_of_type(entries: object, entry_type: str) -> bool: + try: + rows: Final = _OBJECT_LIST.validate_python(entries) + except ValidationError: + return False + return any(_entry_type(row) == entry_type for row in rows) + + +def requests_compaction(context_management: object) -> bool: + if _has_entry_of_type(context_management, "compaction"): + return True + anthropic_form: Final = _json_object(context_management) + return anthropic_form is not None and _has_entry_of_type(anthropic_form.get("edits"), "compact_20260112") + + +def provider_takes_compaction_blocks(model: str, custom_llm_provider: str | None) -> bool: + from litellm.llms.bedrock.claude_platform.transformation import BedrockClaudePlatformConfig + + chat_config: Final = provider_chat_config(model, custom_llm_provider) + return chat_config_builds_anthropic_request(chat_config) and not isinstance( + chat_config, BedrockClaudePlatformConfig + ) + + +def _without_affinity_wrap(encrypted_content: str) -> str: + _, original = ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id(encrypted_content) + return original + + +def decode_compaction_item(encrypted_content: object) -> CompactionBlock | None: + if not isinstance(encrypted_content, str): + return None + try: + decoded: Final = _JSON_OBJECT.validate_json(_without_affinity_wrap(encrypted_content)) + except ValidationError: + return None + return _compaction_block(decoded) + + +def compaction_summary(item: object) -> str | None: + fields: Final = _json_object(item) + if fields is None or fields.get("type") != "compaction": + return None + block: Final = decode_compaction_item(fields.get("encrypted_content")) + return None if block is None else block.summary + + +def summary_only_compaction_content(summary: str) -> str: + return json.dumps({"type": "compaction", "content": summary}, separators=(",", ":")) + + +def with_compaction_summary(encrypted_content: str, summary: str) -> str: + original: Final = _without_affinity_wrap(encrypted_content) + block: Final = decode_compaction_item(original) + if block is None: + return encrypted_content + rewritten: Final = json.dumps({**block.wire_block, "content": summary}, separators=(",", ":")) + return encrypted_content.removesuffix(original) + rewritten + + +def with_visited_compaction_summary(item: object, visit: Callable[[str], str]) -> Mapping[str, object] | None: + fields: Final = _json_object(item) + if fields is None: + return None + summary: Final = compaction_summary(fields) + encrypted_content: Final = fields.get("encrypted_content") + if summary is None or not isinstance(encrypted_content, str): + return None + return {**fields, "encrypted_content": with_compaction_summary(encrypted_content, visit(summary))} + + +def holds_compaction_summary(input: object) -> bool: + try: + items: Final = _OBJECT_LIST.validate_python(input) + except ValidationError: + return False + return any(compaction_summary(item) is not None for item in items) + + +def _is_instruction_item(item: object) -> bool: + fields: Final = _json_object(item) + if fields is None or fields.get("type", "message") != "message": + return False + role: Final = fields.get("role") + return isinstance(role, str) and role in _INSTRUCTION_ROLES + + +def input_items(input: object) -> tuple[object, ...]: + try: + return tuple(_OBJECT_LIST.validate_python(input)) + except ValidationError: + return () + + +def instruction_items(input: object) -> tuple[object, ...]: + return tuple(item for item in input_items(input) if _is_instruction_item(item)) + + +def _call_id(item: object) -> str | None: + fields: Final = _json_object(item) + call_id: Final = None if fields is None else fields.get("call_id") + return call_id if isinstance(call_id, str) else None + + +def _is_turn_item(item: object) -> bool: + fields: Final = _json_object(item) + item_type: Final = None if fields is None else fields.get("type", "message") + if fields is None or not isinstance(item_type, str): + return False + if item_type == "message": + return fields.get("role") == "assistant" + return item_type == "reasoning" or (_call_id(fields) is not None and not item_type.endswith("_output")) + + +def _turn_indices_answered_later( + turn: tuple[tuple[int, object], ...], calls_answered_later: frozenset[int] +) -> tuple[int, ...]: + if not any(index in calls_answered_later for index, _ in turn): + return () + return tuple(index for index, item in turn if _call_id(item) is None or index in calls_answered_later) + + +def _is_turn_entry(entry: tuple[int, object]) -> bool: + return _is_turn_item(entry[1]) + + +def _calls_answered_only_later(earlier: Sequence[object], answered: frozenset[str]) -> frozenset[int]: + last_index_by_call_id: Final = MappingProxyType( + {call_id: index for index, item in enumerate(earlier) if (call_id := _call_id(item)) is not None} + ) + return frozenset(index for call_id, index in last_index_by_call_id.items() if call_id in answered) + + +def _earlier_turn_indices_answered_later(earlier: Sequence[object], answered: frozenset[str]) -> tuple[int, ...]: + calls_answered_later: Final = _calls_answered_only_later(earlier, answered) + turns: Final = (tuple(turn) for is_turn, turn in groupby(enumerate(earlier), key=_is_turn_entry) if is_turn) + return tuple(chain.from_iterable(_turn_indices_answered_later(turn, calls_answered_later) for turn in turns)) + + +def _call_ids(items: Sequence[object]) -> frozenset[str]: + return frozenset(call_id for item in items if (call_id := _call_id(item)) is not None) + + +def _keeps_every_item(item: object) -> bool: + return True + + +def _with_answered_turns_after_newest_compaction( + input: Sequence[_Item], keeps_earlier_item: Callable[[object], bool] +) -> tuple[_Item, ...]: + newest: Final = next( + (index for index in range(len(input) - 1, 0, -1) if compaction_summary(input[index]) is not None), 0 + ) + later: Final = input[newest:] + moved: Final = _earlier_turn_indices_answered_later(input[:newest], _call_ids(later)) + moved_indices: Final = frozenset(moved) + moved_call_ids: Final = _call_ids([input[index] for index in moved]) + first_answer: Final = next( + (index for index, item in enumerate(later) if _call_id(item) in moved_call_ids), len(later) + ) + return ( + *(item for index, item in enumerate(input[:newest]) if index not in moved_indices and keeps_earlier_item(item)), + *later[:first_answer], + *(input[index] for index in moved), + *later[first_answer:], + ) + + +def _items_from_newest_compaction(input: Sequence[_Item]) -> tuple[_Item, ...]: + return _with_answered_turns_after_newest_compaction(input, _is_instruction_item) + + +def with_answered_turns_after_newest_compaction(input: str | ResponseInputParam) -> str | ResponseInputParam: + return ( + list(_with_answered_turns_after_newest_compaction(input, _keeps_every_item)) + if isinstance(input, list) + else input + ) + + +def without_items_before_compaction(input: str | ResponseInputParam) -> str | ResponseInputParam: + return list(_items_from_newest_compaction(input)) if isinstance(input, list) else input + + +def _summary_or_item(item: object) -> object: + summary: Final = compaction_summary(item) + return item if summary is None else summary + + +def billable_input_since_newest_compaction(input: object) -> object: + try: + items: Final = _OBJECT_LIST.validate_python(input) + except ValidationError: + return input + return [_summary_or_item(item) for item in _items_from_newest_compaction(items)] + + +def _as_summary_message(item: _Item) -> _Item | EasyInputMessageParam: + summary: Final = compaction_summary(item) + return item if summary is None else EasyInputMessageParam(role="user", content=summary, type="message") + + +def with_compaction_items_as_summaries(input: str | ResponseInputParam) -> str | ResponseInputParam: + if not isinstance(input, list) or not holds_compaction_summary(input): + return input + return [_as_summary_message(item) for item in _items_from_newest_compaction(input)] + + +def messages_since_newest_compaction(input: object) -> object: + try: + items: Final = _OBJECT_LIST.validate_python(input) + except ValidationError: + return input + return [_as_summary_message(item) for item in _items_from_newest_compaction(items)] diff --git a/litellm/responses/litellm_completion_transformation/count_request.py b/litellm/responses/litellm_completion_transformation/count_request.py index 56a0b9024c5..7679b4ede71 100644 --- a/litellm/responses/litellm_completion_transformation/count_request.py +++ b/litellm/responses/litellm_completion_transformation/count_request.py @@ -1,4 +1,4 @@ -from collections.abc import Mapping +from collections.abc import Awaitable, Callable, Mapping from dataclasses import dataclass from typing import Final, cast # noqa: TID251 # input items are forwarded as sent, as inference does @@ -14,7 +14,11 @@ from litellm.litellm_core_utils.tool_fields import DropPolicy from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body from litellm.responses.additional_tools import hoist_additional_tools from litellm.responses.file_search.emulated_handler import replace_file_search_tools -from litellm.responses.litellm_completion_transformation.transformation import LiteLLMCompletionResponsesConfig +from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary +from litellm.responses.litellm_completion_transformation.transformation import ( + ChatCompletionSession, + LiteLLMCompletionResponsesConfig, +) from litellm.responses.main import previous_response_client_ids from litellm.types.llms.openai import ResponseInputParam from litellm.utils import ProviderConfigManager @@ -132,12 +136,17 @@ def _with_emulated_file_search(responses_request: Mapping[str, object]) -> Mappi return {**responses_request, "tools": replace_file_search_tools(_ITEMS.validate_python(tools))[0]} +def _with_compaction_as_summary_text(responses_request: Mapping[str, object]) -> Mapping[str, object]: + return {name: value for name, value in responses_request.items() if name != "context_management"} + + async def _with_previous_turns( chat_body: Mapping[str, object], model: str, custom_llm_provider: str, responses_request: Mapping[str, object], policy: DropPolicy, + session_lookup: Callable[[str], Awaitable[ChatCompletionSession]] | None, ) -> Mapping[str, object]: previous_response_id: Final = responses_request.get("previous_response_id") if not isinstance(previous_response_id, str) or not previous_response_id: @@ -147,8 +156,10 @@ async def _with_previous_turns( previous_response_id=previous_response_id, litellm_completion_request={**chat_body, "model": model, "custom_llm_provider": custom_llm_provider}, instructions=instructions if isinstance(instructions, str) else None, + session_lookup=session_lookup, request_kwargs={"drop_params": policy.drop_params}, previous_response_client_ids=previous_response_client_ids(previous_response_id, responses_request), + input_replaces_history=holds_compaction_summary(responses_request.get("input")), ) @@ -171,6 +182,7 @@ async def chat_count_request( api_base: str | None, responses_request: Mapping[str, object], policy: DropPolicy, + session_lookup: Callable[[str], Awaitable[ChatCompletionSession]] | None = None, ) -> ChatCountRequest | ChatCountBridgeError | None: """The chat request Responses inference sends a provider with no native Responses API, or None when inference sends this provider the Responses body itself.""" @@ -184,7 +196,9 @@ async def chat_count_request( try: chat_body: Final = await _with_previous_turns( responses_batch_body_to_chat_body( - _with_emulated_file_search(_with_hoisted_additional_tools(responses_request)), + _with_compaction_as_summary_text( + _with_emulated_file_search(_with_hoisted_additional_tools(responses_request)) + ), custom_llm_provider, policy.drop_params, ), @@ -192,6 +206,7 @@ async def chat_count_request( custom_llm_provider, responses_request, policy, + session_lookup, ) tools: Final = ( () if "tools" in policy.additional_drop_params else tuple(map(_unmarked, _mappings(chat_body.get("tools")))) diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 5489d2223ac..c9d800421f7 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -9,11 +9,13 @@ from pydantic import TypeAdapter import litellm from litellm.responses.additional_tools import hoist_additional_tools +from litellm.responses.litellm_completion_transformation.compaction_items import holds_compaction_summary from litellm.responses.litellm_completion_transformation.streaming_iterator import ( LiteLLMCompletionStreamingIterator, ) from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, + with_compaction_items_as_params, ) from litellm.responses.streaming_iterator import BaseResponsesAPIStreamingIterator from litellm.types.llms.openai import ( @@ -120,6 +122,7 @@ class LiteLLMCompletionTransformationHandler: instructions=responses_api_request.get("instructions"), request_kwargs=_REQUEST_KWARGS_ADAPTER.validate_python(kwargs), previous_response_client_ids=previous_response_client_ids, + input_replaces_history=holds_compaction_summary(with_compaction_items_as_params(request_input)), ) acompletion_args: Final = {} diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index 35160830fdc..d50e9a61edc 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -1,12 +1,20 @@ import asyncio import json from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from itertools import chain from typing import TYPE_CHECKING, Any, Final, cast import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER from litellm.litellm_core_utils.cold_storage_handler import ColdStorageHandler +from litellm.responses.litellm_completion_transformation.compaction_items import ( + holds_compaction_summary, + input_items, + instruction_items, + response_holds_compaction_block, +) from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ( AllMessageValues, @@ -20,6 +28,7 @@ if TYPE_CHECKING: from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload from litellm.responses.litellm_completion_transformation.transformation import ( ChatCompletionSession, + SessionMessage, ) else: ChatCompletionSession = Any @@ -66,6 +75,95 @@ def _stored_responses_api_output(stored_response: object) -> ResponseInputParam return None +@dataclass(frozen=True, slots=True) +class _ReplayStart: + index: int + at_output: bool + + +def _compaction_point( + index: int, spend_log: "SpendLogsPayload", proxy_server_request: Mapping[str, object] | None +) -> _ReplayStart | None: + if response_holds_compaction_block(spend_log.get("response")): + return _ReplayStart(index=index, at_output=True) + if proxy_server_request is not None and holds_compaction_summary(proxy_server_request.get("input")): + return _ReplayStart(index=index, at_output=False) + return None + + +def _replay_start( + spend_logs: Sequence["SpendLogsPayload"], proxy_server_requests: Sequence[Mapping[str, object] | None] +) -> _ReplayStart: + return next( + ( + start + for index in range(len(spend_logs) - 1, -1, -1) + if (start := _compaction_point(index, spend_logs[index], proxy_server_requests[index])) is not None + ), + _ReplayStart(index=0, at_output=False), + ) + + +def _holds_compaction_point( + spend_logs: Sequence["SpendLogsPayload"], proxy_server_requests: Sequence[Mapping[str, object] | None] +) -> bool: + return any( + _compaction_point(index, spend_log, proxy_server_request) is not None + for index, (spend_log, proxy_server_request) in enumerate(zip(spend_logs, proxy_server_requests, strict=True)) + ) + + +def _referenced_request_ids(previous_response_id: str, client_response_ids: tuple[str, ...]) -> tuple[str, ...]: + decoded_response_id: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(previous_response_id) + response_id: Final = decoded_response_id.get("response_id", previous_response_id) + return tuple(dict.fromkeys((response_id, previous_response_id, *client_response_ids))) + + +def _turns_through_referenced_response( + spend_logs: Sequence["SpendLogsPayload"], referenced_request_ids: tuple[str, ...] +) -> int: + return next( + ( + index + 1 + for index in range(len(spend_logs) - 1, -1, -1) + if spend_logs[index].get("request_id") in referenced_request_ids + ), + len(spend_logs), + ) + + +def _replayed_request( + index: int, + proxy_server_request: Mapping[str, object] | None, + start: _ReplayStart, + earlier_instructions: tuple[object, ...], +) -> Mapping[str, object] | None: + if proxy_server_request is None or index != start.index: + return proxy_server_request + stored_input: Final = proxy_server_request.get("input") + if start.at_output: + return { + **proxy_server_request, + "input": [*earlier_instructions, *instruction_items(stored_input)], + "messages": None, + } + if not earlier_instructions: + return proxy_server_request + return {**proxy_server_request, "input": [*earlier_instructions, *input_items(stored_input)]} + + +def _instruction_items_before( + proxy_server_requests: Sequence[Mapping[str, object] | None], start: _ReplayStart +) -> tuple[object, ...]: + return tuple( + chain.from_iterable( + instruction_items(request.get("input")) + for request in proxy_server_requests[: start.index] + if request is not None + ) + ) + + def _stored_instructions(proxy_server_request: Mapping[str, object] | None) -> str | None: instructions: Final = None if proxy_server_request is None else proxy_server_request.get("instructions") return instructions if isinstance(instructions, str) and instructions else None @@ -91,6 +189,7 @@ class ResponsesSessionHandler: """ from litellm.responses.litellm_completion_transformation.transformation import ( ChatCompletionSession, + with_stored_calls_the_input_answers, ) verbose_proxy_logger.debug("inside get_chat_completion_message_history_for_previous_response_id") @@ -107,26 +206,34 @@ class ResponsesSessionHandler: if len(all_spend_logs) > 0: litellm_session_id = all_spend_logs[0].get("session_id") - chat_completion_message_history: list[ - AllMessageValues - | GenericChatCompletionMessage - | ChatCompletionMessageToolCall - | ChatCompletionResponseMessage - | Message - ] = [] proxy_server_requests: Final = [ await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log) for spend_log in all_spend_logs ] - for spend_log, proxy_server_request_dict in zip(all_spend_logs, proxy_server_requests): - chat_completion_message_history = ( - ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload( - spend_log=spend_log, - proxy_server_request_dict=proxy_server_request_dict, - chat_completion_message_history=chat_completion_message_history, - item_cache_control=item_cache_control, - ) - ) + replayable_turns: Final = _turns_through_referenced_response( + all_spend_logs, _referenced_request_ids(previous_response_id, client_response_ids) + ) + start: Final = _replay_start(all_spend_logs[:replayable_turns], proxy_server_requests[:replayable_turns]) + replayed_turns: Final = ( + replayable_turns if _holds_compaction_point(all_spend_logs, proxy_server_requests) else len(all_spend_logs) + ) + earlier_instructions: Final = _instruction_items_before(proxy_server_requests, start) + replayed_history: Final = ResponsesSessionHandler._history_of_turns( + all_spend_logs[start.index : replayed_turns], + [ + _replayed_request(index, proxy_server_requests[index], start, earlier_instructions) + for index in range(start.index, replayed_turns) + ], + item_cache_control, + ) + history_before_replay: Final = ResponsesSessionHandler._history_of_turns( + all_spend_logs[: start.index], proxy_server_requests[: start.index], item_cache_control + ) + chat_completion_message_history: Final = ( + list(with_stored_calls_the_input_answers(replayed_history, history_before_replay)) + if history_before_replay + else replayed_history + ) verbose_proxy_logger.debug( "chat_completion_message_history %s", @@ -138,6 +245,24 @@ class ResponsesSessionHandler: instructions=_latest_replayable_instructions(proxy_server_requests), ) + @staticmethod + def _history_of_turns( + spend_logs: Sequence["SpendLogsPayload"], + proxy_server_requests: Sequence[Mapping[str, object] | None], + item_cache_control: bool, + ) -> list["SessionMessage"]: + return list( + chain.from_iterable( + ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload( + spend_log=spend_log, + proxy_server_request_dict=proxy_server_request, + chat_completion_message_history=[], + item_cache_control=item_cache_control, + ) + for spend_log, proxy_server_request in zip(spend_logs, proxy_server_requests, strict=True) + ) + ) + @staticmethod def extend_chat_completion_message_with_spend_log_payload( spend_log: "SpendLogsPayload", @@ -182,6 +307,7 @@ class ResponsesSessionHandler: responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, + only_since_compaction=True, ) ) @@ -196,6 +322,7 @@ class ResponsesSessionHandler: responses_api_request={}, replay_reasoning=True, item_cache_control=item_cache_control, + only_since_compaction=True, ) ) elif isinstance(_response_output, dict) and _response_output and _response_output != {}: @@ -342,12 +469,11 @@ class ResponsesSessionHandler: verbose_proxy_logger.debug("decoding response id=%s", previous_response_id) - decoded_response_id: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(previous_response_id) - response_id: Final = decoded_response_id.get("response_id", previous_response_id) + request_ids: Final = list(_referenced_request_ids(previous_response_id, client_response_ids)) + response_id: Final = request_ids[0] if prisma_client is None: return [] - request_ids: Final = list(dict.fromkeys((response_id, previous_response_id, *client_response_ids))) query: Final = """ WITH matching_session AS ( SELECT session_id, api_key, "user", team_id diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index 4ef68aa072a..c25dd4a405e 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -8,6 +8,7 @@ import litellm from litellm.litellm_core_utils.hidden_params import get_or_create_hidden_params, served_from_cache from litellm.litellm_core_utils.litellm_logging import BilledRoundsSpend from litellm.main import stream_chunk_builder +from litellm.responses.litellm_completion_transformation.compaction_items import compaction_output_item from litellm.responses.litellm_completion_transformation.custom_tools import ( build_tool_call_item_kwargs, extract_custom_tool_names, @@ -83,6 +84,11 @@ def _output_items_with_id(items: tuple[Any, ...], item_type: str, item_id: str | ) +def _streams_compaction(chunk: ModelResponseStream) -> bool: + delta_fields: Final = chunk.choices[0].delta.provider_specific_fields if chunk.choices else None + return delta_fields is not None and "compaction_blocks" in delta_fields + + def _delta_has_signed_thinking_block(delta: object) -> bool: blocks: Final = getattr(delta, "thinking_blocks", None) or () return any(isinstance(b, dict) and (b.get("signature") or b.get("data")) for b in blocks) @@ -146,6 +152,9 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._cached_item_id: str | None = None self._message_output_index: int | None = None self._reasoning_output_index: int | None = None + self._compaction_output_index: int | None = None + self._compaction_item_id: str | None = None + self._sent_compaction_item_done: bool = False self._cached_response_id: str | None = None self._buffered_chunk: ModelResponseStream | None = None self._upstream_exhausted: bool = False @@ -186,7 +195,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._tool_search_call_ids = self._tool_search_call_ids | {call_id} return True - def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None: + def _next_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> OutputItemAddedEvent: self._sequence_number += 1 added: Final = OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, @@ -194,7 +203,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): item=item, ) added.__dict__["sequence_number"] = self._sequence_number - self._pending_tool_events.append(added) + return added + + def _queue_output_item_added(self, output_index: int, item: BaseLiteLLMOpenAIResponseObject) -> None: + self._pending_tool_events.append(self._next_output_item_added(output_index, item)) def _queue_tool_search_call_added(self, call_id: str, output_index: int, args_delta: str) -> None: if call_id not in self._tool_args_by_call_id: @@ -248,6 +260,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return self._message_output_index case "reasoning": return self._reasoning_output_index + case "compaction": + return self._compaction_output_index case _: call_id: Final = getattr(item, "call_id", None) or str(getattr(item, "id", "")).removeprefix("ws_") return self._tool_output_index_by_call_id.get(str(call_id)) @@ -256,6 +270,45 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): index: Final = self._streamed_output_index(item) return (index is None, index or 0) + def _streamed_compaction_item(self) -> BaseLiteLLMOpenAIResponseObject | None: + item: Final = compaction_output_item(self._accumulated_provider_specific_fields) + if item is None: + return None + self._compaction_item_id = self._compaction_item_id or item.id + return BaseLiteLLMOpenAIResponseObject(**{**item.model_dump(exclude_none=True), "id": self._compaction_item_id}) + + def _queue_compaction_item_added(self) -> None: + if ( + self._compaction_output_index is not None + or "compaction_blocks" not in self._accumulated_provider_specific_fields + ): + return + streamed_item: Final = self._streamed_compaction_item() + if streamed_item is None: + return + self._compaction_output_index = self._allocate_output_index() + unfinished_item: Final = streamed_item.model_copy(update={"encrypted_content": ""}) + self._pending_response_events.append( + self._next_output_item_added(self._compaction_output_index, unfinished_item) + ) + + def _queue_compaction_item_done(self) -> None: + if self._compaction_output_index is None or self._sent_compaction_item_done: + return + streamed_item: Final = self._streamed_compaction_item() + if streamed_item is None: + return + self._sent_compaction_item_done = True + self._sequence_number += 1 + self._pending_response_events.append( + OutputItemDoneEvent( + type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, + output_index=self._compaction_output_index, + sequence_number=self._sequence_number, + item=streamed_item, + ) + ) + def _normalize_tool_call_index(self, tool_call: object) -> int | None: idx_raw: Final = tool_call.get("index") if isinstance(tool_call, dict) else getattr(tool_call, "index", None) if idx_raw is None: @@ -592,6 +645,12 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): } ) + def _ordered_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream: + ordered: Final = self._reasoning_part_first(chunk) + if not _streams_compaction(ordered): + self._queue_compaction_item_added() + return ordered + def _with_encoded_response_id(self, response: ResponsesAPIResponse) -> ResponsesAPIResponse: return ResponsesAPIRequestUtils.update_responses_api_response_id_with_model_id( responses_api_response=response, @@ -1004,6 +1063,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if not self.litellm_model_response or isinstance(self.litellm_model_response, TextCompletionResponse): self.litellm_model_response = self.create_litellm_model_response() if self.litellm_model_response: + self._queue_compaction_item_added() + self._queue_compaction_item_done() + if self._pending_response_events: + return self._pending_response_events.pop(0) # If tool calls exist, emit tool events before finishing/response.completed. if isinstance(self.litellm_model_response, ModelResponse): self._queue_final_tool_call_done_events(self.litellm_model_response) @@ -1091,6 +1154,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._reasoning_item_id or self._cached_reasoning_item_id or mint_reasoning_item_id() ) reasoning_item_id: Final = self._cached_reasoning_item_id + self._queue_compaction_item_done() self._sequence_number += 1 text_done_event: Final = self.create_reasoning_summary_text_done_event( reasoning_item_id=reasoning_item_id, @@ -1139,7 +1203,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): raise StopAsyncIteration chunk = await self.litellm_custom_stream_wrapper.__anext__() if chunk is not None: - chunk = self._reasoning_part_first(cast(ModelResponseStream, chunk)) + chunk = self._ordered_chunk(cast(ModelResponseStream, chunk)) for src in ( getattr(chunk, "provider_specific_fields", None), getattr( @@ -1195,11 +1259,11 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): try: buffered_chunk = self._take_buffered_chunk() if buffered_chunk is not None: - chunk = self._reasoning_part_first(buffered_chunk) + chunk = self._ordered_chunk(buffered_chunk) elif self._upstream_exhausted: raise StopIteration else: - chunk = self._reasoning_part_first(self.litellm_custom_stream_wrapper.__next__()) + chunk = self._ordered_chunk(self.litellm_custom_stream_wrapper.__next__()) for src in ( getattr(chunk, "provider_specific_fields", None), getattr( @@ -1356,7 +1420,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "reasoning", self._cached_reasoning_item_id, ) - streamed_items: Final = filterfalse(self._is_unstreamed_empty_message, reasoning_aligned) + compaction_aligned: Final = _output_items_with_id(reasoning_aligned, "compaction", self._compaction_item_id) + streamed_items: Final = filterfalse(self._is_unstreamed_empty_message, compaction_aligned) return tuple(sorted(streamed_items, key=self._streamed_output_position)) def _is_unstreamed_empty_message(self, item: object) -> bool: @@ -1388,6 +1453,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): messages=LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped input=self.request_input, responses_api_request=dict(self.responses_api_request), + only_since_compaction=True, ), logging_obj=self.litellm_logging_obj, ) diff --git a/litellm/responses/litellm_completion_transformation/tool_search.py b/litellm/responses/litellm_completion_transformation/tool_search.py index 0a94acdd461..780a23e02b2 100644 --- a/litellm/responses/litellm_completion_transformation/tool_search.py +++ b/litellm/responses/litellm_completion_transformation/tool_search.py @@ -1,5 +1,5 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence, Set -from typing import Final, Literal, TypeAlias +from typing import TYPE_CHECKING, Final, Literal, TypeAlias from pydantic import ConfigDict, TypeAdapter, ValidationError @@ -12,6 +12,9 @@ from litellm.types.utils import LlmProviders, Message from .custom_tools import openai_shaped_tool_call_item_id +if TYPE_CHECKING: + from litellm.llms.base_llm.chat.transformation import BaseConfig + __all__ = [ "ANTHROPIC_TOOL_SEARCH_TOOL_NAME", "ANTHROPIC_TOOL_SEARCH_TOOL_TYPE", @@ -31,6 +34,7 @@ __all__ = [ "is_openai_tool_search_tool", "is_server_tool_search_tool", "provider_builds_anthropic_request", + "provider_chat_config", "runs_tool_search_on_the_server", "tool_search_call_ids", "tools_with_anthropic_tool_search_mapped", @@ -159,7 +163,7 @@ def input_without_unanswered_anthropic_tool_search_calls( return input if len(kept) == len(input) else kept -def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool: +def provider_chat_config(model: str, custom_llm_provider: str | None) -> "BaseConfig | None": from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider from litellm.utils import ProviderConfigManager @@ -167,10 +171,12 @@ def provider_builds_anthropic_request(model: str, custom_llm_provider: str | Non try: llm_provider: Final = LlmProviders(provider) except ValueError: - return False - return chat_config_builds_anthropic_request( - ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) - ) + return None + return ProviderConfigManager.get_provider_chat_config(model=model, provider=llm_provider) + + +def provider_builds_anthropic_request(model: str, custom_llm_provider: str | None) -> bool: + return chat_config_builds_anthropic_request(provider_chat_config(model, custom_llm_provider)) def chat_config_builds_anthropic_request(chat_config: object) -> bool: diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index d9ebf50cc1a..02d9566d396 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -8,7 +8,7 @@ import uuid from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass from functools import partial -from itertools import accumulate, chain, islice +from itertools import accumulate, chain, groupby, islice from types import MappingProxyType from typing import ( TYPE_CHECKING, @@ -29,6 +29,7 @@ from openai.types.chat.chat_completion_named_tool_choice_param import ( Function as NamedToolChoiceFunction, ) from openai.types.responses import ResponseFunctionToolCall, ResponseFunctionWebSearch +from openai.types.responses.response_compaction_item import ResponseCompactionItem from openai.types.responses.response_create_params import ResponseInputParam from openai.types.responses.tool_choice_custom_param import ToolChoiceCustomParam from openai.types.responses.tool_choice_function_param import ToolChoiceFunctionParam @@ -120,6 +121,17 @@ from litellm.types.utils import ( Usage, ) +from .compaction_items import ( + carries_only_compaction, + compaction_output_item, + decode_compaction_item, + holds_compaction_summary, + provider_compaction_summary, + provider_takes_compaction_blocks, + requests_compaction, + with_answered_turns_after_newest_compaction, + without_items_before_compaction, +) from .custom_tools import ( convert_custom_tool_to_function_tool, extract_custom_tool_names, @@ -224,6 +236,7 @@ TOOL_CALLS_CACHE: Final = InMemoryCache() _ANY_KEY_DICT_ADAPTER: Final = TypeAdapter(dict[object, object]) _STR_KEY_DICT_ADAPTER: Final = TypeAdapter(dict[str, object]) _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object]) +_MessageT = TypeVar("_MessageT") _CACHED_CONTENT_ADAPTER: Final = TypeAdapter(ChatCompletionCachedContent) _DICT_ITEMS_LIST_ADAPTER: Final = TypeAdapter(list[dict[object, object]]) _TEXT_ADAPTER: Final = TypeAdapter(str) @@ -306,6 +319,126 @@ def _without_stored_tool_search_uses(message: SessionMessage, search_call_ids: f return message +def _instruction_messages(messages: Sequence[SessionMessage]) -> tuple[SessionMessage, ...]: + return tuple( + message + for message in messages + if isinstance(message, Mapping) and message.get("role") in ("system", "developer") + ) + + +def _message_field(message: object, key: str) -> object: + if isinstance(message, Mapping): + return _STR_KEY_DICT_ADAPTER.validate_python(message).get(key) + return getattr(message, key, None) + + +def _str_field(message: object, key: str) -> str | None: + value: Final = _message_field(message, key) + return value if isinstance(value, str) else None + + +def _tool_output_call_id(message: object) -> str | None: + return _str_field(message, "tool_call_id") if _message_field(message, "role") == "tool" else None + + +def _assistant_tool_calls(message: object) -> tuple[object, ...]: + if _message_field(message, "role") != "assistant": + return () + return tuple(_OBJECT_LIST_ADAPTER.validate_python(_message_field(message, "tool_calls") or [])) + + +def _answered_tool_call_ids(messages: Sequence[object]) -> frozenset[str]: + called: Final = frozenset( + _str_field(call, "id") for call in chain.from_iterable(map(_assistant_tool_calls, messages)) + ) + return frozenset( + call_id for call_id in map(_tool_output_call_id, messages) if call_id is not None and call_id not in called + ) + + +def _stored_turn_with_calls(message: SessionMessage, call_ids: frozenset[str]) -> SessionMessage | None: + calls: Final = [call for call in _assistant_tool_calls(message) if _str_field(call, "id") in call_ids] + if not calls: + return None + if isinstance(message, Message): + return message.model_copy( + update={"tool_calls": calls, "provider_specific_fields": _provider_fields_without_compaction(message)} + ) + return cast( # cast-ok: the stored assistant turn keeps every field but the calls the new input leaves unanswered + SessionMessage, {**_STR_KEY_DICT_ADAPTER.validate_python(message), "tool_calls": calls} + ) + + +def _call_ids_at_turn(indexed_message: tuple[int, object]) -> tuple[tuple[str | None, int], ...]: + index, message = indexed_message + return tuple((_str_field(call, "id"), index) for call in _assistant_tool_calls(message)) + + +def _newest_stored_calls_the_input_answers( + session_history: Sequence[SessionMessage], answered: frozenset[str] +) -> Iterator[SessionMessage | None]: + newest_turn_by_call_id: Final = MappingProxyType( + dict(chain.from_iterable(map(_call_ids_at_turn, enumerate(session_history)))) + ) + answered_turns: Final = tuple( + sorted((newest_turn_by_call_id[call_id], call_id) for call_id in answered if call_id in newest_turn_by_call_id) + ) + answered_by_turn: Final = MappingProxyType( + { + turn: frozenset(call_id for _, call_id in calls) + for turn, calls in groupby(answered_turns, key=lambda turn_and_call: turn_and_call[0]) + } + ) + return ( + _stored_turn_with_calls(message, answered_by_turn.get(index, frozenset())) + for index, message in enumerate(session_history) + ) + + +def with_stored_calls_the_input_answers( + new_messages: Sequence[_MessageT], session_history: Sequence[SessionMessage] +) -> tuple[_MessageT | SessionMessage, ...]: + answered: Final = _answered_tool_call_ids(new_messages) + stored_calls: Final = tuple( + turn for turn in _newest_stored_calls_the_input_answers(session_history, answered) if turn is not None + ) + if not stored_calls: + return tuple(new_messages) + first_output: Final = next( + index for index, message in enumerate(new_messages) if _tool_output_call_id(message) in answered + ) + return (*new_messages[:first_output], *stored_calls, *new_messages[first_output:]) + + +def _stored_compaction_summary(message: SessionMessage) -> str | None: + return provider_compaction_summary(message.provider_specific_fields) if isinstance(message, Message) else None + + +def _provider_fields_without_compaction(message: Message) -> Mapping[str, object] | None: + if not message.provider_specific_fields: + return None + other_fields: Final = { + key: value + for key, value in _STR_KEY_DICT_ADAPTER.validate_python(message.provider_specific_fields).items() + if key != "compaction_blocks" + } + return other_fields or None + + +def _with_stored_compaction_as_summary(message: SessionMessage) -> tuple[SessionMessage, ...]: + if not isinstance(message, Message): + return (message,) + summary: Final = provider_compaction_summary(message.provider_specific_fields) + if summary is None: + return (message,) + answer: Final = message.model_copy( + update={"provider_specific_fields": _provider_fields_without_compaction(message)} + ) + summary_message: Final = ChatCompletionUserMessage(role="user", content=summary) + return (summary_message, answer) if answer.content or answer.tool_calls else (summary_message,) + + def _input_item_cache_control(input_item: Mapping[str, object]) -> ChatCompletionCachedContent | None: cache_control: Final = input_item.get("cache_control") if not cache_control: @@ -340,6 +473,32 @@ def without_item_cache_control(input: str | ResponseInputParam) -> str | Respons ) +def _as_input_param(item: object) -> object: + return item.model_dump(exclude_none=True) if isinstance(item, ResponseCompactionItem) else item + + +def with_compaction_items_as_params(input: str | ResponseInputParam) -> str | ResponseInputParam: + if not isinstance(input, list): + return input + return cast( # cast-ok: the same input items, a compaction item replayed from SDK output dumped to its param dict + ResponseInputParam, [_as_input_param(item) for item in input] + ) + + +def _input_to_read( + input: str | ResponseInputParam, only_since_compaction: bool, native_compaction: bool +) -> str | ResponseInputParam: + if only_since_compaction: + return without_items_before_compaction(input) + if native_compaction: + return with_answered_turns_after_newest_compaction(input) + return input + + +def _replays_compaction_block(input: str | ResponseInputParam, context_management: object) -> bool: + return requests_compaction(context_management) and holds_compaction_summary(input) + + _PROVIDERS_KEEPING_MESSAGE_CACHE_CONTROL: Final = frozenset(("anthropic", "dashscope", "minimax", "sap", "zai")) @@ -426,9 +585,6 @@ def _block_with_cache_control(block: object, rewrite: _CacheControlRewrite) -> o return _fields_with_cache_control(_STR_KEY_DICT_ADAPTER.validate_python(block), rewrite) -_MessageT = TypeVar("_MessageT") - - def _with_cache_control(message: _MessageT, rewrite: _CacheControlRewrite) -> _MessageT: if not isinstance(message, dict): return message @@ -870,14 +1026,21 @@ class LiteLLMCompletionResponsesConfig: ) request_kwargs: Final = _STR_KEY_DICT_ADAPTER.validate_python({**kwargs, "drop_params": drop_params}) + context_management: Final = responses_api_request.get("context_management") keeps_item_marks: Final = _target_keeps_item_marks( model, custom_llm_provider, drops_unsupported_params(request_kwargs.get("drop_params")) ) + request_input: Final = with_compaction_items_as_params(input) + native_compaction: Final = _replays_compaction_block( + request_input, context_management + ) and provider_takes_compaction_blocks(model, custom_llm_provider) bridged_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=input, + input=request_input, responses_api_request=responses_api_request, replay_reasoning=True, item_cache_control=keeps_item_marks, + native_compaction=native_compaction, + only_since_compaction=not native_compaction, ) litellm_completion_request: dict = { @@ -909,7 +1072,7 @@ class LiteLLMCompletionResponsesConfig: "response_format": response_format, "reasoning_effort": reasoning.effort, "reasoning_summary": reasoning.summary, - "context_management": responses_api_request.get("context_management"), + "context_management": context_management, # litellm specific params "custom_llm_provider": custom_llm_provider, "extra_headers": extra_headers, @@ -939,6 +1102,8 @@ class LiteLLMCompletionResponsesConfig: responses_api_request: ResponsesAPIOptionalRequestParams | dict, replay_reasoning: bool = False, item_cache_control: bool = False, + native_compaction: bool = False, + only_since_compaction: bool = False, ) -> list[ AllMessageValues | GenericChatCompletionMessage @@ -960,6 +1125,19 @@ class LiteLLMCompletionResponsesConfig: guardrail scanning) leave both off, because they need every piece of text in the request to stay readable as message ``content``, with no item mark a strict message schema could refuse. + + ``native_compaction`` belongs to callers whose target builds an + Anthropic request that also asks for compaction in + ``context_management``: a ``compaction`` item then goes back as the + provider's own compaction block, which Anthropic rejects without that + compaction strategy. Everyone else reads its summary as a user message. + + ``only_since_compaction`` belongs to callers that send, count, or + rebuild what the model reads: the items before the newest + ``compaction`` item are dropped, keeping system and developer + messages, the way Anthropic ignores content before a compaction block. + Guardrail scans leave it off because they write rewrites back by input + item position. """ messages: list[ AllMessageValues @@ -975,10 +1153,13 @@ class LiteLLMCompletionResponsesConfig: ) ) + dumped_input: Final = with_compaction_items_as_params(input) + read_input: Final = _input_to_read(dumped_input, only_since_compaction, native_compaction) messages.extend( LiteLLMCompletionResponsesConfig._transform_response_input_param_to_chat_completion_message( - input=input if item_cache_control else without_item_cache_control(input), + input=read_input if item_cache_control else without_item_cache_control(read_input), replay_reasoning=replay_reasoning, + native_compaction=native_compaction, ) ) @@ -997,6 +1178,18 @@ class LiteLLMCompletionResponsesConfig: if (kept := _without_stored_tool_search_uses(message, search_call_ids)) is not None ) + @staticmethod + def _with_compaction_blocks_the_route_takes( + messages: Sequence[SessionMessage], route: CompletionRoute, context_management: object + ) -> tuple[SessionMessage, ...]: + if not any(_stored_compaction_summary(message) is not None for message in messages): + return tuple(messages) + if requests_compaction(context_management) and provider_takes_compaction_blocks( + route.model, route.custom_llm_provider + ): + return tuple(messages) + return tuple(chain.from_iterable(map(_with_stored_compaction_as_summary, messages))) + @staticmethod async def async_responses_api_session_handler( previous_response_id: str, @@ -1005,6 +1198,7 @@ class LiteLLMCompletionResponsesConfig: session_lookup: Callable[[str], Awaitable[ChatCompletionSession]] | None = None, request_kwargs: Mapping[str, object] = MappingProxyType({}), previous_response_client_ids: tuple[str, ...] = (), + input_replaces_history: bool = False, ) -> dict[str, object]: """ Async hook to get the chain of previous input and output pairs and return a list of Chat Completion messages @@ -1027,11 +1221,15 @@ class LiteLLMCompletionResponsesConfig: else ChatCompletionSession(messages=[], litellm_session_id=None) ) _messages: Final = litellm_completion_request.get("messages") or [] - stored_messages: Final = chat_completion_session.get("messages") or [] + route: Final = CompletionRoute(model=model, custom_llm_provider=provider) + session_history: Final = chat_completion_session.get("messages") or [] + stored_messages: Final = LiteLLMCompletionResponsesConfig._with_compaction_blocks_the_route_takes( + _instruction_messages(session_history) if input_replaces_history else session_history, + route, + request_fields.get("context_management"), + ) replayed_messages: Final = ( - LiteLLMCompletionResponsesConfig._without_tool_search_uses_off_anthropic( - stored_messages, CompletionRoute.model_validate(litellm_completion_request) - ) + LiteLLMCompletionResponsesConfig._without_tool_search_uses_off_anthropic(stored_messages, route) if any(holds_tool_search_use(message) for message in stored_messages) else stored_messages ) @@ -1062,11 +1260,16 @@ class LiteLLMCompletionResponsesConfig: # Store original _messages before combining for safety check original_new_messages: Final = _messages.copy() if _messages else [] - combined_messages = [*leading_system_messages, *session_messages, *_messages[instructions_end:]] + new_messages: Final = ( + with_stored_calls_the_input_answers(_messages[instructions_end:], session_history) + if input_replaces_history + else _messages[instructions_end:] + ) + combined_messages = [*leading_system_messages, *session_messages, *new_messages] # Fix: Ensure tool_results have corresponding tool_calls in previous assistant message # Pass tools parameter to help reconstruct tool_calls if not in cache - tools: Final = litellm_completion_request.get("tools") or [] + tools: Final = _OBJECT_LIST_ADAPTER.validate_python(litellm_completion_request.get("tools") or []) combined_messages = LiteLLMCompletionResponsesConfig._ensure_tool_results_have_corresponding_tool_calls( messages=combined_messages, tools=tools ) @@ -1115,6 +1318,7 @@ class LiteLLMCompletionResponsesConfig: def _transform_response_input_param_to_chat_completion_message( input: str | ResponseInputParam, replay_reasoning: bool = False, + native_compaction: bool = False, ) -> list[ AllMessageValues | GenericChatCompletionMessage | ChatCompletionMessageToolCall | ChatCompletionResponseMessage ]: @@ -1140,6 +1344,7 @@ class LiteLLMCompletionResponsesConfig: LiteLLMCompletionResponsesConfig._transform_responses_api_input_item_to_chat_completion_message( input_item=_input, replay_reasoning=replay_reasoning, + native_compaction=native_compaction, ) ) @@ -1523,7 +1728,7 @@ class LiteLLMCompletionResponsesConfig: """Find the index of the previous assistant message.""" for j in range(current_idx - 1, -1, -1): if messages[j].get("role") == "assistant": - return j + return None if carries_only_compaction(messages[j]) else j return None @staticmethod @@ -1839,6 +2044,7 @@ class LiteLLMCompletionResponsesConfig: def _transform_responses_api_input_item_to_chat_completion_message( input_item: Mapping[str, object], replay_reasoning: bool = False, + native_compaction: bool = False, ) -> list[AllMessageValues | GenericChatCompletionMessage | ChatCompletionResponseMessage]: """ Transform a Responses API input item into a Chat Completion message @@ -1877,6 +2083,12 @@ class LiteLLMCompletionResponsesConfig: Mapping[str, str], input_item ) ) + elif input_item.get("type") == "compaction": + return list( + LiteLLMCompletionResponsesConfig._transform_compaction_item_to_chat_completion_messages( + input_item, native_compaction + ) + ) elif input_item.get("type") == "reasoning": # A ResponseReasoningItemParam carries the prior-turn chain-of-thought. # Chat-completions providers (DeepSeek V4, Kimi K2.6, ...) expect this @@ -1928,6 +2140,23 @@ class LiteLLMCompletionResponsesConfig: return [] return [LiteLLMCompletionResponsesConfig._transform_responses_api_message_item(input_item, content)] + @staticmethod + def _transform_compaction_item_to_chat_completion_messages( + input_item: Mapping[str, object], native_compaction: bool + ) -> tuple[ChatCompletionUserMessage | ChatCompletionResponseMessage, ...]: + compaction: Final = decode_compaction_item(input_item.get("encrypted_content")) + if compaction is None: + return () + if native_compaction: + return ( + ChatCompletionResponseMessage( + role="assistant", + content=None, + provider_specific_fields={"compaction_blocks": [dict(compaction.wire_block)]}, + ), + ) + return (ChatCompletionUserMessage(role="user", content=compaction.summary),) + @staticmethod def _transform_responses_api_message_item( input_item: Mapping[str, object], content: object @@ -3038,6 +3267,7 @@ class LiteLLMCompletionResponsesConfig: | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall + | ResponseCompactionItem ]: responses_output: list[ GenericResponseOutputItem @@ -3048,8 +3278,10 @@ class LiteLLMCompletionResponsesConfig: | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall + | ResponseCompactionItem ] = [] + responses_output.extend(LiteLLMCompletionResponsesConfig._extract_compaction_output_items(choices)) responses_output.extend( LiteLLMCompletionResponsesConfig._extract_reasoning_output_items(chat_completion_response, choices) ) @@ -3115,6 +3347,11 @@ class LiteLLMCompletionResponsesConfig: output_items.append(item) return output_items + @staticmethod + def _extract_compaction_output_items(choices: Sequence[Choices]) -> tuple[ResponseCompactionItem, ...]: + item: Final = compaction_output_item(choices[0].message.provider_specific_fields) if choices else None + return () if item is None else (item,) + @staticmethod def _encode_thinking_blocks(message: Message) -> str | None: thinking_blocks: Final[Sequence[Mapping[str, object]]] = getattr(message, "thinking_blocks", None) or () diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 3bdb1c9c375..09eea7cb323 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -40,9 +40,15 @@ from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_h from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig +from litellm.responses.litellm_completion_transformation.compaction_items import ( + with_compaction_items_as_summaries, +) from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, ) +from litellm.responses.litellm_completion_transformation.transformation import ( + with_compaction_items_as_params, +) from litellm.responses.mcp.request_context import MCPRequestContext from litellm.responses.mcp.round_billing import ( billed_for_every_round, @@ -1669,7 +1675,9 @@ def responses( ) # Decode any litellm-encoded encrypted-content item IDs back to their original IDs - input = ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input) + input = with_compaction_items_as_summaries( + with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input)) + ) # Call the handler with _is_async flag instead of directly calling the async handler if custom_llm_provider is None: @@ -2516,7 +2524,9 @@ def compact_responses( # Decode any litellm-encoded encrypted-content item IDs back to their original IDs # before forwarding to the upstream provider. - input = ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input) + input = with_compaction_items_as_summaries( + with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input)) + ) # Call the handler with _is_async flag instead of directly calling the async handler response = base_llm_http_handler.compact_response_api_handler( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index bc30da72df9..0d59c93786c 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -1707,6 +1707,7 @@ def _estimate_usage_from_text( messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( # pyright: ignore[reportUnknownMemberType] # the transformer's signature is partially untyped input=request_input, # pyright: ignore[reportArgumentType] # the raw Responses API input is a str or ResponseInputParam list, matching the helper's declared union responses_api_request=dict(responses_api_request), + only_since_compaction=True, ) input_tokens: Final = litellm.token_counter( # pyright: ignore[reportUnknownMemberType] # token_counter's public signature is untyped model=model, messages=messages diff --git a/litellm/router.py b/litellm/router.py index fe418b5830b..da04e59f18a 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -14069,6 +14069,7 @@ class Router: input_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( input=typed_input, responses_api_request={"instructions": instructions} if instructions is not None else {}, + only_since_compaction=True, ) return litellm.token_counter( messages=cast(list, input_messages), # cast-ok: transformed chat messages diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 0ed156524be..089f21a33c1 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -386,6 +386,9 @@ class AgenticLoopState: @dataclass(frozen=True, slots=True, kw_only=True) class RouterState: weights: "RouterWeights | None" = field(default=None, metadata=wire("_router_weights")) + encrypted_content_affinity_pinned: bool | None = field( + default=None, metadata=wire("_encrypted_content_affinity_pinned") + ) fallback_depth: int | None = None max_fallbacks: int | None = None attempted_targets: "AttemptedFallbackTargets | None" = None diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index 6aeead7679f..ba422a0a901 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -481,6 +481,11 @@ class ContentThinkingSignatureBlockDelta(TypedDict): signature: str +class ContentCompactionBlockDelta(TypedDict): + type: ReadOnly[Literal["compaction_delta"]] + content: ReadOnly[str | None] + + StreamingContentBlockDeltaType = Literal["text_delta", "input_json_delta", "thinking_delta", "signature_delta"] @@ -493,6 +498,7 @@ class ContentBlockDelta(TypedDict): | ContentCitationsBlockDelta | ContentThinkingBlockDelta | ContentThinkingSignatureBlockDelta + | ContentCompactionBlockDelta ) diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 8dbc0d97d54..62dc152ca84 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -55,6 +55,7 @@ except (ImportError, AttributeError): from typing import Annotated +from openai.types.responses.response_compaction_item import ResponseCompactionItem from openai.types.responses.response_create_params import ( Reasoning, ResponseIncludable, @@ -1488,6 +1489,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): | ResponseFunctionWebSearch | CustomToolCallOutputItem | OutputToolSearchCall + | ResponseCompactionItem ] ) parallel_tool_calls: bool | None = None diff --git a/tests/benchmarks/test_responses_compaction_benchmarks.py b/tests/benchmarks/test_responses_compaction_benchmarks.py new file mode 100644 index 00000000000..62a5e95255e --- /dev/null +++ b/tests/benchmarks/test_responses_compaction_benchmarks.py @@ -0,0 +1,35 @@ +import json +from collections.abc import Mapping +from typing import Final + +import pytest + +from litellm.responses.litellm_completion_transformation.compaction_items import ( + with_answered_turns_after_newest_compaction, +) + +COMPACTION_ITEM: Final[Mapping[str, object]] = { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": "Turn one planned Tokyo."}), +} +LONG_TURN_OF_ASSISTANT_TEXT: Final[tuple[Mapping[str, object], ...]] = tuple( + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": f"Step {index}."}]} + for index in range(5_000) +) +CALL: Final[Mapping[str, object]] = { + "type": "function_call", + "call_id": "call_osaka", + "name": "get_weather", + "arguments": "{}", +} +OUTPUT: Final[Mapping[str, object]] = {"type": "function_call_output", "call_id": "call_osaka", "output": "Sunny."} + + +@pytest.mark.benchmark +def test_with_answered_turns_after_newest_compaction_long_turn_split_from_its_output() -> None: + replayed: Final = with_answered_turns_after_newest_compaction( + [*LONG_TURN_OF_ASSISTANT_TEXT, CALL, COMPACTION_ITEM, OUTPUT] + ) + + assert replayed == [COMPACTION_ITEM, *LONG_TURN_OF_ASSISTANT_TEXT, CALL, OUTPUT] diff --git a/tests/integration/providers/test_responses_compaction_items_wire.py b/tests/integration/providers/test_responses_compaction_items_wire.py new file mode 100644 index 00000000000..5b8e2b534d7 --- /dev/null +++ b/tests/integration/providers/test_responses_compaction_items_wire.py @@ -0,0 +1,1057 @@ +import json +import os +import re +import uuid +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from pathlib import Path +from typing import Final, Literal, TypeAlias +from urllib.parse import urlsplit + +import httpx +import openai +import pytest +import yaml +from integration._support import claude_code as cc +from integration._support.client import ( + Gateway, + Scenario, + eventually, + gateway_from_environment, + object_value, + string_value, +) +from integration._support.database import read_rows +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from integration._support.worker_readiness import wait_for_worker_readiness +from pydantic import JsonValue, TypeAdapter + +from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD +from litellm.responses.utils import ResponsesAPIRequestUtils + +pytestmark: Final = [pytest.mark.timeout(2 * graceful_stop_seconds() + 120), pytest.mark.shares_worker_state] + +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_CLAUDE: Final = cc.OPUS +_GEMINI: Final = "gemini-3.8-flash" +_OPENAI: Final = "gpt-5.6" +_PROXY_WORKERS: Final = max(1, int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1"))) +_CACHE_BUST: Final[Mapping[str, JsonValue]] = {"cache": {"no-cache": True}} +_COMPACTING: Final[list[JsonValue]] = [{"type": "compaction", "compact_threshold": 150000}] +_COMPACT_BETA: Final = "compact-2026-01-12" +_CITY: Final = "Tokyo" +_MASKED_CITY: Final = "[CITY]" +_MASK_INPUT: Final = "compaction-mask-input" +_MASK_OUTPUT: Final = "compaction-mask-output" +_BLOCK_OUTPUT: Final = "compaction-block-output" +_REDACT_HEADER: Final = {"x-litellm-enable-message-redaction": "true"} +_UNREDACT_HEADER: Final = {"litellm-disable-message-redaction": "true"} +_SCRIPTED_REJECTION: Final = json.dumps( + {"type": "error", "error": {"type": "invalid_request_error", "message": "scripted rejection"}} +).encode() +_ASK: Final = re.compile(r"ask-([0-9a-f]{32})") + +Route: TypeAlias = Literal["claude-compacting", "claude-plain", "gemini", "openai", "openai-compact", "openai-opaque"] + + +def _new_marker() -> str: + return uuid.uuid4().hex + + +def _ask(marker: str, *, compact: bool = False) -> str: + return f"ask-{marker} compact-now-{marker}" if compact else f"ask-{marker}" + + +def _fail(marker: str) -> str: + return f"fail-now-{marker}" + + +def _think(marker: str) -> str: + return f"think-now-{marker}" + + +def _thinking(marker: str) -> dict[str, JsonValue]: + return {"type": "thinking", "thinking": f"thought-{marker}", "signature": f"signature-{marker}"} + + +def _answer(marker: str) -> str: + return f"answer-{marker}" + + +def _summary(marker: str) -> str: + return f"kept-{marker} {'a' * 1500} middle-{marker} {'b' * 1500} the user lives in {_CITY}" + + +def _block(summary: str) -> dict[str, JsonValue]: + return {"type": "compaction", "content": summary} + + +def _compaction_item(summary: str) -> dict[str, JsonValue]: + return { + "type": "compaction", + "id": f"cmp_{uuid.uuid4().hex}", + "encrypted_content": json.dumps(_block(summary), separators=(",", ":")), + } + + +def _newest_marker(raw: bytes) -> str: + found: Final = _ASK.findall(raw.decode()) + assert found, raw[:400] + return str(found[-1]) + + +def _anthropic_usage() -> dict[str, JsonValue]: + return {"input_tokens": 20, "output_tokens": 5} + + +def _anthropic_block_frames(index: int, block: Mapping[str, JsonValue]) -> tuple[bytes, ...]: + def start(content_block: Mapping[str, JsonValue]) -> bytes: + return cc.sse_frame( + "content_block_start", {"type": "content_block_start", "index": index, "content_block": dict(content_block)} + ) + + def delta(fields: Mapping[str, JsonValue]) -> bytes: + return cc.sse_frame( + "content_block_delta", {"type": "content_block_delta", "index": index, "delta": dict(fields)} + ) + + stop: Final = cc.sse_frame("content_block_stop", {"type": "content_block_stop", "index": index}) + match block["type"]: + case "compaction": + summary: Final = string_value(block["content"]) + half: Final = len(summary) // 2 + return ( + start({"type": "compaction", "content": ""}), + delta({"type": "compaction_delta", "content": summary[:half]}), + delta({"type": "compaction_delta", "content": summary[half:]}), + stop, + ) + case "thinking": + return ( + start({"type": "thinking", "thinking": ""}), + delta({"type": "thinking_delta", "thinking": block["thinking"]}), + delta({"type": "signature_delta", "signature": block["signature"]}), + stop, + ) + case _: + return (start({"type": "text", "text": ""}), delta({"type": "text_delta", "text": block["text"]}), stop) + + +def _anthropic_stream(identity: str, content: Sequence[Mapping[str, JsonValue]]) -> tuple[bytes, ...]: + start: Final = cc.sse_frame( + "message_start", + { + "type": "message_start", + "message": { + "id": identity, + "type": "message", + "role": "assistant", + "model": _CLAUDE, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 20, "output_tokens": 1}, + }, + }, + ) + blocks: Final = tuple( + chain.from_iterable(_anthropic_block_frames(index, block) for index, block in enumerate(content)) + ) + end: Final = cc.sse_frame( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 5}, + }, + ) + return (start, *blocks, end, cc.sse_frame("message_stop", {"type": "message_stop"})) + + +def _claude_reply(body: Mapping[str, JsonValue], raw: bytes) -> Reply: + marker: Final = _newest_marker(raw) + if _fail(marker).encode() in raw: + return Reply(status=400, body=_SCRIPTED_REJECTION) + compaction: Final = (_block(_summary(marker)),) if f"compact-now-{marker}".encode() in raw else () + thinking: Final = (_thinking(marker),) if _think(marker).encode() in raw else () + content: Final = (*compaction, *thinking, {"type": "text", "text": _answer(marker)}) + identity: Final = f"msg_{uuid.uuid4().hex}" + if body.get("stream") is True: + return Reply(content_type="text/event-stream", chunks=_anthropic_stream(identity, content)) + return Reply(body=cc.message_reply(identity, _CLAUDE, content, _anthropic_usage())) + + +def _gemini_reply(raw: bytes) -> Reply: + frame: Final = { + "responseId": f"gemini-{uuid.uuid4().hex}", + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": _answer(_newest_marker(raw))}]}, + "index": 0, + "finishReason": "STOP", + } + ], + "usageMetadata": {"promptTokenCount": 20, "candidatesTokenCount": 5, "totalTokenCount": 25}, + "modelVersion": _GEMINI, + } + return Reply(body=json.dumps(frame).encode()) + + +_OPENAI_USAGE: Final = { + "input_tokens": 20, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 5, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 25, +} + + +def _openai_reply(raw: bytes) -> Reply: + tag: Final = uuid.uuid4().hex + message: Final = { + "id": f"msg_{tag}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": _answer(_newest_marker(raw)), "annotations": []}], + } + response: Final = { + "id": f"resp_{tag}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": _OPENAI, + "output": [message], + "usage": _OPENAI_USAGE, + } + return Reply(body=json.dumps(response).encode()) + + +def _openai_compact_reply() -> Reply: + compacted: Final = { + "id": f"resp_compact_{uuid.uuid4().hex}", + "created_at": 1, + "object": "response.compaction", + "output": [], + "usage": _OPENAI_USAGE, + } + return Reply(body=json.dumps(compacted).encode()) + + +def _vendor(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=json.dumps({"object": "list", "data": [{"id": _OPENAI, "object": "model"}]}).encode()) + path: Final = urlsplit(request.target).path + if path.endswith("/v1/messages/count_tokens"): + return Reply(body=json.dumps({"input_tokens": _word_count(_JSON_OBJECT.validate_json(request.body))}).encode()) + if path.endswith("/v1/messages"): + return _claude_reply(_JSON_OBJECT.validate_json(request.body), request.body) + if path.endswith(":generateContent"): + return _gemini_reply(request.body) + if path.endswith("/responses/compact"): + return _openai_compact_reply() + assert path.endswith("/responses"), request.target + return _openai_reply(request.body) + + +def _provider_calls(wire: Wire, marker: str) -> tuple[Request, ...]: + needle: Final = f"ask-{marker}".encode() + return tuple(request for request in wire.drain() if request.method == "POST" and needle in request.body) + + +def _only_call(wire: Wire, marker: str) -> tuple[Request, dict[str, JsonValue]]: + calls: Final = _provider_calls(wire, marker) + assert len(calls) == 1, [call.target for call in calls] + return calls[0], _JSON_OBJECT.validate_json(calls[0].body) + + +def _objects(value: JsonValue) -> tuple[dict[str, JsonValue], ...]: + assert isinstance(value, list), value + return tuple(object_value(item) for item in value) + + +def _message_texts(message: Mapping[str, JsonValue], key: str = "content") -> tuple[str, ...]: + content: Final = message.get(key) + if isinstance(content, str): + return (content,) + if not isinstance(content, list): + return () + return tuple(string_value(part["text"]) for part in _objects(content) if isinstance(part.get("text"), str)) + + +def _anthropic_texts(body: Mapping[str, JsonValue], role: str) -> tuple[str, ...]: + messages: Final = tuple(message for message in _objects(body["messages"]) if message["role"] == role) + return tuple(chain.from_iterable(map(_message_texts, messages))) + + +def _word_count(body: Mapping[str, JsonValue]) -> int: + texts: Final = (*_anthropic_texts(body, "user"), *_anthropic_texts(body, "assistant")) + return sum(len(text.split()) for text in texts) + + +def _anthropic_block_types(body: Mapping[str, JsonValue]) -> tuple[JsonValue, ...]: + contents: Final = tuple(message["content"] for message in _objects(body["messages"])) + blocks: Final = chain.from_iterable(_objects(content) for content in contents if isinstance(content, list)) + return tuple(block.get("type") for block in blocks) + + +def _gemini_texts(body: Mapping[str, JsonValue], role: str) -> tuple[str, ...]: + turns: Final = tuple(turn for turn in _objects(body["contents"]) if turn["role"] == role) + return tuple(chain.from_iterable(_message_texts(turn, "parts") for turn in turns)) + + +def _openai_user_texts(body: Mapping[str, JsonValue]) -> tuple[str, ...]: + items: Final = tuple(item for item in _objects(body["input"]) if item.get("role") == "user") + return tuple(chain.from_iterable(map(_message_texts, items))) + + +def _openai_item_types(body: Mapping[str, JsonValue]) -> tuple[JsonValue, ...]: + return tuple(item.get("type") for item in _objects(body["input"])) + + +def _register(scenario: Scenario, wire: Wire, provider: Literal["claude", "gemini", "openai"]) -> str: + match provider: + case "claude": + model = scenario.model(model=f"anthropic/{_CLAUDE}", api_base=wire.url, api_key=cc.ANTHROPIC_API_KEY) + case "gemini": + model = scenario.model(model=f"gemini/{_GEMINI}", api_base=wire.url, api_key="synthetic-gemini-key") + case "openai": + model = scenario.model(model=f"openai/{_OPENAI}", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key") + wait_for_worker_readiness(scenario.gateway, model, 1, _PROXY_WORKERS) + return model + + +def _openai_client(gateway: Gateway) -> openai.OpenAI: + return openai.OpenAI( + base_url=f"{gateway.client.base_url}/v1", + api_key=gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False, timeout=60), + ) + + +def _decoded(encrypted_content: str) -> dict[str, JsonValue]: + return _JSON_OBJECT.validate_json(encrypted_content) + + +def _responses( + gateway: Gateway, body: Mapping[str, JsonValue], headers: Mapping[str, str] | None = None +) -> dict[str, JsonValue]: + response: Final = gateway.request("POST", "/v1/responses", {**body, **_CACHE_BUST}, headers=headers) + assert response.status_code == 200, response.text + return _JSON_OBJECT.validate_json(response.content) + + +def _sse_events(text: str) -> tuple[dict[str, JsonValue], ...]: + payloads: Final = tuple( + line.removeprefix("data:").strip() for line in text.splitlines() if line.startswith("data:") + ) + return tuple(_JSON_OBJECT.validate_json(payload) for payload in payloads if payload != "[DONE]") + + +def _stream( + gateway: Gateway, path: str, body: Mapping[str, JsonValue], headers: Mapping[str, str] | None = None +) -> tuple[dict[str, JsonValue], ...]: + sent_headers: Final = {"Authorization": f"Bearer {gateway.key}", **(headers or {})} + with gateway.client.stream( + "POST", path, json={**body, **_CACHE_BUST, "stream": True}, headers=sent_headers + ) as response: + text: Final = response.read().decode() + assert response.status_code == 200, text + return _sse_events(text) + + +def _delta_provider_fields(chunk: Mapping[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]: + return tuple( + fields + for choice in _objects(chunk.get("choices", [])) + if isinstance(fields := object_value(choice.get("delta", {})).get("provider_specific_fields"), dict) + ) + + +def _streamed_compaction_blocks(chunks: Sequence[Mapping[str, JsonValue]]) -> tuple[JsonValue, ...]: + fields: Final = chain.from_iterable(_delta_provider_fields(chunk) for chunk in chunks) + return tuple(field["compaction_blocks"] for field in fields if "compaction_blocks" in field) + + +def _output_items(response: Mapping[str, JsonValue], item_type: str) -> tuple[dict[str, JsonValue], ...]: + return tuple(item for item in _objects(response["output"]) if item["type"] == item_type) + + +def _listed(value: JsonValue) -> list[JsonValue]: + assert isinstance(value, list), value + return value + + +def _only(items: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]: + assert len(items) == 1, items + return items[0] + + +@pytest.mark.parametrize("stream", (False, True), ids=("json", "stream")) +def test_claude_compaction_block_is_returned_as_a_compaction_output_item(gateway: Gateway, stream: bool) -> None: + marker: Final = _new_marker() + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _register(scenario, wire, "claude") + client: Final = _openai_client(gateway) + request: Final = { + "input": _ask(marker, compact=True), + "extra_body": {"context_management": _COMPACTING, **_CACHE_BUST}, + } + if not stream: + reply: Final = client.responses.create(model=model, **request) + assert [item.type for item in reply.output] == ["compaction", "message"] + compaction: Final = reply.output[0] + assert compaction.type == "compaction" + assert _decoded(compaction.encrypted_content) == _block(_summary(marker)) + assert reply.output_text == _answer(marker) + else: + events: Final = tuple(client.responses.create(model=model, stream=True, **request)) + added: Final = [e for e in events if e.type == "response.output_item.added" and e.item.type == "compaction"] + done: Final = [e for e in events if e.type == "response.output_item.done" and e.item.type == "compaction"] + completed: Final = [e for e in events if e.type == "response.completed"] + assert len(added) == 1 and len(done) == 1 and len(completed) == 1, [e.type for e in events] + assert added[0].item.encrypted_content == "" + assert done[0].output_index == added[0].output_index + assert done[0].item.id == added[0].item.id + assert _decoded(done[0].item.encrypted_content) == _block(_summary(marker)) + finished: Final = [item for item in completed[0].response.output if item.type == "compaction"] + assert [(item.id, item.encrypted_content) for item in finished] == [ + (done[0].item.id, done[0].item.encrypted_content) + ] + message_indexes: Final = { + e.output_index for e in events if e.type == "response.output_item.added" and e.item.type == "message" + } + assert message_indexes and added[0].output_index not in message_indexes + assert completed[0].response.output_text == _answer(marker) + call, body = _only_call(wire, marker) + assert _COMPACT_BETA in call.headers.get("anthropic-beta", ""), dict(call.headers) + + +def _history(marker: str, compaction: Mapping[str, JsonValue]) -> list[JsonValue]: + return [ + {"role": "developer", "content": f"rules-{marker}"}, + {"role": "user", "content": f"old-{marker}"}, + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": f"old-answer-{marker}", "annotations": []}], + }, + dict(compaction), + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": f"recap-{marker}", "annotations": []}], + }, + {"role": "user", "content": _ask(marker)}, + ] + + +_ROUTES: Final[tuple[Route, ...]] = ( + "claude-compacting", + "claude-plain", + "gemini", + "openai", + "openai-compact", + "openai-opaque", +) + + +def _route_provider(route: Route) -> Literal["claude", "gemini", "openai"]: + if route.startswith("claude"): + return "claude" + return "gemini" if route == "gemini" else "openai" + + +def _assert_reached( + route: Route, marker: str, sent: Mapping[str, JsonValue], call: Request, body: Mapping[str, JsonValue] +) -> None: + summary: Final = _summary(marker) + dumped: Final = json.dumps(body) + if route == "openai-opaque": + assert urlsplit(call.target).path.endswith("/responses"), call.target + assert sent in _objects(body["input"]), body["input"] + assert f"old-{marker}" in dumped + return + if route != "claude-compacting": + assert f"old-{marker}" not in dumped and f"old-answer-{marker}" not in dumped, dumped[:2000] + assert f"rules-{marker}" in dumped + assert f"recap-{marker}" in dumped + match route: + case "claude-compacting": + assistant: Final = tuple( + message for message in _objects(body["messages"]) if message["role"] == "assistant" + ) + blocks: Final = tuple( + chain.from_iterable( + _objects(message["content"]) for message in assistant if isinstance(message["content"], list) + ) + ) + assert _block(summary) in blocks, body["messages"] + assert summary not in _anthropic_texts(body, "user") + assert f"rules-{marker}" in json.dumps(body["system"]) + assert _COMPACT_BETA in call.headers.get("anthropic-beta", ""), dict(call.headers) + case "claude-plain": + assert "compaction" not in _anthropic_block_types(body), body["messages"] + assert summary in _anthropic_texts(body, "user"), body["messages"] + case "gemini": + assert summary in _gemini_texts(body, "user"), body["contents"] + case "openai" | "openai-compact": + suffix: Final = "/responses/compact" if route == "openai-compact" else "/responses" + assert urlsplit(call.target).path.endswith(suffix), call.target + assert "compaction" not in _openai_item_types(body), body["input"] + assert summary in _openai_user_texts(body), body["input"] + + +@pytest.mark.parametrize("route", _ROUTES) +def test_compaction_item_reaches_each_route_in_the_shape_it_reads(gateway: Gateway, route: Route) -> None: + marker: Final = _new_marker() + compaction: Final = ( + {"type": "compaction", "id": f"cmp_{marker}", "encrypted_content": f"gAAAAA-opaque-{marker}"} + if route == "openai-opaque" + else _compaction_item(_summary(marker)) + ) + compacting: Final[dict[str, JsonValue]] = ( + {"context_management": _COMPACTING} if route == "claude-compacting" else {} + ) + dropping: Final[dict[str, JsonValue]] = ( + {"context_management": _COMPACTING, "drop_params": True} if route == "gemini" else {} + ) + path: Final = "/v1/responses/compact" if route == "openai-compact" else "/v1/responses" + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _register(scenario, wire, _route_provider(route)) + response: Final = gateway.request( + "POST", + path, + {"model": model, "input": _history(marker, compaction), **compacting, **dropping, **_CACHE_BUST}, + ) + assert response.status_code == 200, response.text + if route != "openai-compact": + assert _answer(marker) in response.text, response.text + call, body = _only_call(wire, marker) + _assert_reached(route, marker, compaction, call, body) + + +def _input_tokens(gateway: Gateway, model: str, items: list[JsonValue]) -> int: + counted: Final = gateway.post("/v1/responses/input_tokens", {"model": model, "input": items}) + tokens: Final = counted["input_tokens"] + assert isinstance(tokens, int), counted + return tokens + + +def test_input_tokens_count_from_the_newest_compaction(gateway: Gateway) -> None: + marker: Final = _new_marker() + summary: Final = _summary(marker) + old_history: Final = " ".join(f"history-{index}" for index in range(600)) + compacted: Final[list[JsonValue]] = [ + {"role": "user", "content": old_history}, + _compaction_item(summary), + {"role": "user", "content": _ask(marker)}, + ] + equivalent: Final[list[JsonValue]] = [ + {"role": "user", "content": summary}, + {"role": "user", "content": _ask(marker)}, + ] + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _register(scenario, wire, "claude") + assert _input_tokens(gateway, model, compacted) == _input_tokens(gateway, model, equivalent) + counted: Final = tuple( + request.body for request in wire.drain() if urlsplit(request.target).path.endswith("/count_tokens") + ) + assert len(counted) == 2, counted + assert all(b"history-" not in body for body in counted), counted[0][:2000] + + +def test_streamed_compaction_item_is_done_before_the_reasoning_after_it(gateway: Gateway) -> None: + marker: Final = _new_marker() + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _register(scenario, wire, "claude") + events: Final = _stream( + gateway, + "/v1/responses", + { + "model": model, + "input": f"{_ask(marker, compact=True)} {_think(marker)}", + "context_management": _COMPACTING, + }, + ) + done: Final = tuple( + (position, event["output_index"], object_value(event["item"])["type"]) + for position, event in enumerate(events) + if event["type"] == "response.output_item.done" + ) + completed: Final = _only([event for event in events if event["type"] == "response.completed"]) + assert [(index, kind) for _, index, kind in done] == [(0, "compaction"), (1, "reasoning"), (2, "message")], [ + event["type"] for event in events + ] + assert [item["type"] for item in _objects(object_value(completed["response"])["output"])] == [ + "compaction", + "reasoning", + "message", + ] + reasoning_done: Final = done[1][0] + sequence_numbers: Final = [ + number for event in events[: reasoning_done + 1] if isinstance(number := event.get("sequence_number"), int) + ] + assert sequence_numbers == sorted(set(sequence_numbers)), sequence_numbers + _only_call(wire, marker) + + +def test_chat_stream_reports_the_whole_compaction_block(gateway: Gateway) -> None: + marker: Final = _new_marker() + with wire_server(_vendor) as wire, gateway.scenario() as scenario: + model: Final = _register(scenario, wire, "claude") + chunks: Final = _stream( + gateway, + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": _ask(marker, compact=True)}], + "context_management": {"edits": [{"type": "compact_20260112"}]}, + }, + ) + reported: Final = _streamed_compaction_blocks(chunks) + assert reported, chunks + assert _objects(reported[-1])[-1] == _block(_summary(marker)), reported[-1] + assert len(_provider_calls(wire, marker)) == 1 + + +def _guardrail_peer(request: Request) -> Reply: + payload: Final = _JSON_OBJECT.validate_json(request.body) + texts: Final = payload.get("texts") or [] + assert isinstance(texts, list), payload + if urlsplit(request.target).path.startswith("/block/"): + found: Final = any(isinstance(text, str) and _CITY in text for text in texts) + verdict: Final = {"action": "BLOCKED", "blocked_reason": "city in summary"} if found else {"action": "NONE"} + return Reply(body=json.dumps(verdict).encode()) + masked: Final = [text.replace(_CITY, _MASKED_CITY) if isinstance(text, str) else text for text in texts] + return Reply(body=json.dumps({"action": "GUARDRAIL_INTERVENED", "texts": masked}).encode()) + + +def _guardrail(name: str, mode: str, api_base: str, **fields: JsonValue) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": mode, + "default_on": False, + "api_base": api_base, + "api_key": "synthetic-guardrail-key", + **fields, + }, + } + + +def _deployment(name: str, model: str, api_base: str, **fields: JsonValue) -> dict[str, JsonValue]: + return {"model_name": name, "litellm_params": {"model": model, "api_base": api_base, **fields}} + + +_STORED_CLAUDE: Final = "compaction-stored-claude" +_SMALL_CLAUDE: Final = "compaction-small-claude" +_STORED_GEMINI: Final = "compaction-stored-gemini" +_STORED_OPENAI: Final = "compaction-stored-openai" +_SMALL_CONTEXT: Final = 2000 + + +def _stored_config(directory: Path, provider_url: str, guardrail_url: str, **litellm_settings: JsonValue) -> Path: + stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + claude: Final = {"api_key": cc.ANTHROPIC_API_KEY} + config: Final[dict[str, JsonValue]] = { + **stock, + "model_list": [ + _deployment(_STORED_CLAUDE, f"anthropic/{_CLAUDE}", provider_url, **claude), + { + **_deployment(_SMALL_CLAUDE, f"anthropic/{_CLAUDE}", provider_url, **claude), + "model_info": {"max_input_tokens": _SMALL_CONTEXT}, + }, + _deployment(_STORED_GEMINI, f"gemini/{_GEMINI}", provider_url, api_key="synthetic-gemini-key"), + _deployment(_STORED_OPENAI, f"openai/{_OPENAI}", f"{provider_url}/v1", api_key="synthetic-openai-key"), + ], + "general_settings": { + **object_value(stock["general_settings"]), + "store_prompts_in_spend_logs": True, + "disable_responses_id_security": True, + }, + "litellm_settings": {**object_value(stock["litellm_settings"]), **litellm_settings}, + "router_settings": {**object_value(stock["router_settings"]), "num_retries": 0, "enable_pre_call_checks": True}, + "guardrails": [ + _guardrail(_MASK_INPUT, "pre_call", f"{guardrail_url}/input"), + _guardrail(_MASK_OUTPUT, "post_call", f"{guardrail_url}/output"), + _guardrail(_BLOCK_OUTPUT, "post_call", f"{guardrail_url}/block", streaming_buffer_until_moderated=True), + ], + } + path: Final = directory / "compaction-stored.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@dataclass(frozen=True, slots=True) +class _StoredRig: + gateway: Gateway + provider: Wire + guardrail: Wire + + +def _stored_rig(directory: Path, **litellm_settings: JsonValue) -> Iterator[_StoredRig]: + with ( + wire_server(_vendor) as provider, + wire_server(_guardrail_peer) as guardrail, + gateway_from_environment() as gateway, + ): + config: Final = _stored_config(directory, provider.url, guardrail.url, **litellm_settings) + with owned_proxy(gateway, directory, {}, config=config, workers=2) as owned: + yield _StoredRig(owned, provider, guardrail) + + +@pytest.fixture(scope="module") +def stored(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_StoredRig]: + yield from _stored_rig(tmp_path_factory.mktemp("compaction-stored")) + + +@pytest.fixture(scope="module") +def redacting(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_StoredRig]: + yield from _stored_rig(tmp_path_factory.mktemp("compaction-redacting"), turn_off_message_logging=True) + + +def _logged_row(request_id: str, status: Literal["success", "failure"]) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id, proxy_server_request, response, metadata, status FROM "LiteLLM_SpendLogs"' + " WHERE request_id=%s", + (request_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["status"] == status, rows[0] + return rows[0] + + +def _spend_row(response_id: str) -> dict[str, JsonValue]: + decoded: Final = ResponsesAPIRequestUtils.decode_responses_api_response_id(response_id).get("response_id") + return _logged_row(decoded if isinstance(decoded, str) else response_id, "success") + + +_REPLAY_ROUTES: Final = ( + pytest.param(_STORED_CLAUDE, True, id="claude-compacting"), + pytest.param(_STORED_CLAUDE, False, id="claude-plain"), + pytest.param(_STORED_GEMINI, True, id="gemini"), +) + + +@pytest.mark.parametrize(("follow_up_model", "compacting"), _REPLAY_ROUTES) +def test_previous_response_id_after_a_compaction_turn_replays_the_whole_summary( + stored: _StoredRig, follow_up_model: str, compacting: bool +) -> None: + first: Final = _new_marker() + second: Final = _new_marker() + summary: Final = _summary(first) + long_text: Final = f"long-{first} {'z' * 3000} tail-{first}" + compaction_request: Final[dict[str, JsonValue]] = {"context_management": _COMPACTING} + opened: Final = _responses( + stored.gateway, + { + "model": _STORED_CLAUDE, + "input": [{"role": "user", "content": long_text}, {"role": "user", "content": _ask(first, compact=True)}], + **compaction_request, + }, + ) + assert _decoded(string_value(_only(_output_items(opened, "compaction"))["encrypted_content"])) == _block(summary) + row: Final = _spend_row(string_value(opened["id"])) + assert summary in json.dumps(row["response"]), json.dumps(row["response"])[:600] + stored_request: Final = json.dumps(row["proxy_server_request"]) + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored_request and "z" * 3000 not in stored_request + _only_call(stored.provider, first) + + follow_up: Final = _responses( + stored.gateway, + { + "model": follow_up_model, + "input": _ask(second), + "previous_response_id": opened["id"], + **(compaction_request if compacting else {}), + **({"drop_params": True} if follow_up_model == _STORED_GEMINI else {}), + }, + ) + assert _answer(second) in json.dumps(follow_up) + call, body = _only_call(stored.provider, second) + dumped: Final = json.dumps(body) + assert f"long-{first}" not in dumped and f"ask-{first}" not in dumped, dumped[:2000] + assert _answer(first) in dumped + if follow_up_model == _STORED_GEMINI: + assert summary in _gemini_texts(body, "user"), body["contents"] + elif compacting: + replayed: Final = _objects(body["messages"])[0] + assert replayed["role"] == "assistant", body["messages"] + assert _objects(replayed["content"])[0] == _block(summary), replayed + assert _COMPACT_BETA in call.headers.get("anthropic-beta", ""), dict(call.headers) + else: + assert "compaction" not in _anthropic_block_types(body), body["messages"] + assert summary in _anthropic_texts(body, "user"), body["messages"] + + +def test_router_pre_call_check_counts_from_the_newest_compaction(stored: _StoredRig) -> None: + compacted_marker: Final = _new_marker() + control_marker: Final = _new_marker() + old_history: Final = " ".join(f"history-{index}" for index in range(3 * _SMALL_CONTEXT)) + + def send(marker: str, items: list[JsonValue]) -> httpx.Response: + return stored.gateway.request( + "POST", + "/v1/responses", + {"model": _SMALL_CLAUDE, "input": [*items, {"role": "user", "content": _ask(marker)}], **_CACHE_BUST}, + ) + + compacted: Final = send( + compacted_marker, + [{"role": "user", "content": old_history}, _compaction_item(f"short summary {compacted_marker}")], + ) + assert compacted.status_code == 200, compacted.text + _only_call(stored.provider, compacted_marker) + control: Final = send(control_marker, [{"role": "user", "content": old_history}]) + assert control.status_code == 400, control.text + assert _provider_calls(stored.provider, control_marker) == () + + +_GUARDED_ROUTES: Final = ( + pytest.param(_STORED_OPENAI, id="openai"), + pytest.param(_STORED_CLAUDE, id="claude-compacting"), +) + + +@pytest.mark.parametrize("model", _GUARDED_ROUTES) +def test_pre_call_guardrail_rewrites_the_compaction_summary(stored: _StoredRig, model: str) -> None: + marker: Final = _new_marker() + summary: Final = _summary(marker) + masked: Final = summary.replace(_CITY, _MASKED_CITY) + compacting: Final[dict[str, JsonValue]] = {"context_management": _COMPACTING} if model == _STORED_CLAUDE else {} + _responses( + stored.gateway, + { + "model": model, + "input": [_compaction_item(summary), {"role": "user", "content": _ask(marker)}], + "guardrails": [_MASK_INPUT], + **compacting, + }, + ) + scanned: Final = tuple( + _JSON_OBJECT.validate_json(request.body) + for request in stored.guardrail.drain() + if f"kept-{marker}".encode() in request.body + ) + assert any(summary in _listed(payload["texts"]) for payload in scanned), scanned + call, body = _only_call(stored.provider, marker) + assert _CITY not in json.dumps(body), json.dumps(body)[:2000] + if model == _STORED_OPENAI: + assert masked in _openai_user_texts(body), body["input"] + else: + assert _objects(_objects(body["messages"])[0]["content"])[0] == _block(masked), body["messages"] + + +def test_post_call_guardrail_rewrites_the_returned_compaction_summary(stored: _StoredRig) -> None: + marker: Final = _new_marker() + returned: Final = _responses( + stored.gateway, + { + "model": _STORED_CLAUDE, + "input": _ask(marker, compact=True), + "context_management": _COMPACTING, + "guardrails": [_MASK_OUTPUT], + }, + ) + masked: Final = _block(_summary(marker).replace(_CITY, _MASKED_CITY)) + assert _decoded(string_value(_only(_output_items(returned, "compaction"))["encrypted_content"])) == masked + _only_call(stored.provider, marker) + + +def test_buffered_stream_guardrail_blocks_on_the_compaction_summary(stored: _StoredRig) -> None: + marker: Final = _new_marker() + body: Final[dict[str, JsonValue]] = { + "model": _STORED_CLAUDE, + "input": _ask(marker, compact=True), + "context_management": _COMPACTING, + "guardrails": [_BLOCK_OUTPUT], + "stream": True, + **_CACHE_BUST, + } + headers: Final = {"Authorization": f"Bearer {stored.gateway.key}"} + with stored.gateway.client.stream("POST", "/v1/responses", json=body, headers=headers) as response: + text: Final = response.read().decode() + assert response.status_code == 200, text + peer_calls: Final = tuple( + (urlsplit(request.target).path, _JSON_OBJECT.validate_json(request.body).get("texts")) + for request in stored.guardrail.drain() + if marker.encode() in request.body + ) + assert any(path.startswith("/block/") and _summary(marker) in _listed(texts) for path, texts in peer_calls), [ + (path, [str(value)[:80] for value in _listed(texts or [])]) for path, texts in peer_calls + ] + event_types: Final = tuple(event["type"] for event in _sse_events(text)) + assert "response.failed" in event_types, event_types + assert "response.completed" not in event_types, event_types + assert f"kept-{marker}" not in text and _CITY not in text, text[:2000] + _only_call(stored.provider, marker) + + +def test_redacted_spend_row_holds_no_compaction_summary(stored: _StoredRig) -> None: + marker: Final = _new_marker() + returned: Final = _responses( + stored.gateway, + {"model": _STORED_CLAUDE, "input": _ask(marker, compact=True), "context_management": _COMPACTING}, + headers=_REDACT_HEADER, + ) + assert _decoded(string_value(_only(_output_items(returned, "compaction"))["encrypted_content"])) == _block( + _summary(marker) + ) + row: Final = _spend_row(string_value(returned["id"])) + assert f"kept-{marker}" not in json.dumps(row["response"]), json.dumps(row["response"])[:2000] + _only_call(stored.provider, marker) + + +def test_stored_compaction_input_item_keeps_its_whole_summary_and_nothing_else(stored: _StoredRig) -> None: + first: Final = _new_marker() + second: Final = _new_marker() + summary: Final = _summary(first) + padded_item: Final = { + **_compaction_item(summary), + "encrypted_content": json.dumps({**_block(summary), "padding": f"pad-{first} {'p' * 3000}"}), + } + opened: Final = _responses( + stored.gateway, + { + "model": _STORED_CLAUDE, + "input": [padded_item, {"role": "user", "content": _ask(first)}], + "context_management": _COMPACTING, + }, + ) + stored_input: Final = _objects( + object_value(_spend_row(string_value(opened["id"]))["proxy_server_request"])["input"] + ) + assert _decoded(string_value(stored_input[0]["encrypted_content"])) == _block(summary), stored_input[0] + _only_call(stored.provider, first) + + _responses( + stored.gateway, + { + "model": _STORED_CLAUDE, + "input": _ask(second), + "previous_response_id": opened["id"], + "context_management": _COMPACTING, + }, + ) + _, body = _only_call(stored.provider, second) + assert summary in _anthropic_texts(body, "user"), body["messages"] + assert f"pad-{first}" not in json.dumps(body), body["messages"] + + +def test_redacted_chat_stream_spend_row_holds_no_compaction_delta(stored: _StoredRig) -> None: + marker: Final = _new_marker() + summary: Final = _summary(marker) + chunks: Final = _stream( + stored.gateway, + "/v1/chat/completions", + { + "model": _STORED_CLAUDE, + "messages": [{"role": "user", "content": _ask(marker, compact=True)}], + "context_management": {"edits": [{"type": "compact_20260112"}]}, + }, + headers=_REDACT_HEADER, + ) + streamed_blocks: Final = _streamed_compaction_blocks(chunks) + assert streamed_blocks and _objects(streamed_blocks[-1])[-1] == _block(summary), chunks + secrets: Final = (f"kept-{marker}", summary[len(summary) // 2 :]) + logged: Final = {column: json.dumps(value) for column, value in _spend_row(string_value(chunks[0]["id"])).items()} + leaked: Final = {column: text for column, text in logged.items() if any(secret in text for secret in secrets)} + assert not leaked, {column: text[max(0, text.find(secrets[0]) - 600) :][:1500] for column, text in leaked.items()} + _only_call(stored.provider, marker) + + +def _failing_request(path: str, marker: str) -> dict[str, JsonValue]: + ask: Final = f"{_ask(marker)} {_fail(marker)}" + if path == "/v1/responses": + return { + "model": _STORED_CLAUDE, + "input": [_compaction_item(_summary(marker)), {"role": "user", "content": ask}], + } + return {"model": _STORED_CLAUDE, "max_tokens": 64, "messages": [{"role": "user", "content": ask}]} + + +def _failed_call_row( + rig: _StoredRig, + path: str, + marker: str, + headers: Mapping[str, str], + key: str | None = None, + **fields: JsonValue, +) -> dict[str, JsonValue]: + body: Final = {**_failing_request(path, marker), **fields} + response: Final = rig.gateway.request("POST", path, body, headers=headers, key=key) + assert response.status_code == 400, response.text + _only_call(rig.provider, marker) + return _logged_row(response.headers["x-litellm-call-id"], "failure") + + +_LEGACY_REDACT_HEADER: Final = {"litellm-enable-message-redaction": "true"} +_REDACTED_FAILURES: Final = ( + pytest.param("/v1/responses", _REDACT_HEADER, id="responses"), + pytest.param("/v1/responses", _LEGACY_REDACT_HEADER, id="responses-legacy-header"), + pytest.param("/v1/messages", _REDACT_HEADER, id="messages"), + pytest.param("/v1/chat/completions", _REDACT_HEADER, id="chat"), +) + + +@pytest.mark.parametrize(("path", "headers"), _REDACTED_FAILURES) +def test_failed_call_spend_row_holds_no_request_text_with_the_redaction_header( + stored: _StoredRig, path: str, headers: Mapping[str, str] +) -> None: + marker: Final = _new_marker() + logged: Final = json.dumps(_failed_call_row(stored, path, marker, headers)) + assert marker not in logged, logged[max(0, logged.find(marker) - 600) :][:1500] + + +def test_failed_responses_call_spend_row_keeps_the_whole_compaction_summary_without_the_header( + stored: _StoredRig, +) -> None: + marker: Final = _new_marker() + row: Final = _failed_call_row(stored, "/v1/responses", marker, {}) + stored_input: Final = _objects(object_value(row["proxy_server_request"])["input"]) + assert _decoded(string_value(stored_input[0]["encrypted_content"])) == _block(_summary(marker)), stored_input[0] + assert f"{_ask(marker)} {_fail(marker)}" in _message_texts(stored_input[1]), stored_input[1] + + +def test_failed_responses_call_stores_the_caller_metadata_without_the_request_headers(stored: _StoredRig) -> None: + marker: Final = _new_marker() + row: Final = _failed_call_row(stored, "/v1/responses", marker, _REDACT_HEADER, metadata={"caller": marker}) + stored_metadata: Final = object_value(object_value(row["proxy_server_request"])["metadata"]) + assert stored_metadata.get("caller") == marker, stored_metadata + assert "headers" not in stored_metadata, stored_metadata + + +def test_failed_responses_call_keeps_the_caller_metadata_headers_over_the_redaction_header(stored: _StoredRig) -> None: + marker: Final = _new_marker() + caller_headers: Final = {"caller": marker} + row: Final = _failed_call_row(stored, "/v1/responses", marker, _REDACT_HEADER, metadata={"headers": caller_headers}) + stored_request: Final = object_value(row["proxy_server_request"]) + assert object_value(stored_request["metadata"])["headers"] == caller_headers, stored_request["metadata"] + assert _ask(marker) in json.dumps(stored_request), json.dumps(stored_request)[:1500] + + +_GLOBALLY_REDACTED_FAILURES: Final = ( + pytest.param("/v1/responses", _UNREDACT_HEADER, False, id="responses-disable-header"), + pytest.param("/v1/chat/completions", _UNREDACT_HEADER, True, id="chat-disable-header"), + pytest.param("/v1/responses", {}, False, id="responses-no-header"), +) + + +@pytest.mark.parametrize(("path", "headers", "kept"), _GLOBALLY_REDACTED_FAILURES) +def test_opted_out_key_failed_call_spend_row_takes_the_disable_header_over_global_redaction_only_on_chat( + redacting: _StoredRig, path: str, headers: Mapping[str, str], kept: bool +) -> None: + marker: Final = _new_marker() + with redacting.gateway.scenario() as scenario: + key: Final = scenario.key(metadata={"allow_client_message_redaction_opt_out": True}) + row: Final = _failed_call_row(redacting, path, marker, headers, key) + stored_request: Final = json.dumps(row["proxy_server_request"]) + assert (_ask(marker) in stored_request) is kept, stored_request[:1500] diff --git a/tests/unit/litellm_core_utils/test_redact_messages.py b/tests/unit/litellm_core_utils/test_redact_messages.py index a91888fea79..6a734ca3b70 100644 --- a/tests/unit/litellm_core_utils/test_redact_messages.py +++ b/tests/unit/litellm_core_utils/test_redact_messages.py @@ -57,6 +57,8 @@ def _provider_fields() -> dict[str, object]: "reasoningContentBlocks": [{"reasoningText": {"text": SECRET}}], "citations": [{"cited_text": SECRET}], "thought_signatures": [SIGNATURE], + "compaction_blocks": [{"type": "compaction", "content": SECRET}], + "compaction_delta": {"type": "compaction_delta", "content": SECRET}, } @@ -165,6 +167,26 @@ def test_streaming_redaction_redacts_delta_provider_fields_without_touching_the_ assert fields == _provider_fields() +@pytest.mark.parametrize("shape", ["object", "dict"]) +def test_redaction_hides_a_compaction_summary_in_responses_api_output(shape: str) -> None: + response: Final = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-sonnet-5-5", + object="response", + status="completed", + output=[ + {"type": "compaction", "id": "cmp_1", "encrypted_content": json.dumps({"type": "compaction", "content": SECRET})} + ], + ) + result: Final = response if shape == "object" else response.model_dump() + + redacted: Final = perform_redaction({}, result) + + assert SECRET not in str(redacted), redacted + assert SECRET in str(result) + + @pytest.mark.parametrize("surface", ("typed", "dict", "standard", "callback", "helper")) def test_responses_redaction_removes_instructions_without_changing_the_response(surface: str) -> None: response: Final = litellm.ResponsesAPIResponse.model_validate( diff --git a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py index bf871210074..2f10fd6fd0d 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -34,6 +34,7 @@ from litellm.llms.openai.responses.guardrail_translation.handler import ( from litellm.llms.openai.responses.guardrail_translation.tool_merge import merge_guardrailed_tools from litellm.proxy.guardrails.guardrail_hooks.generic_guardrail_api import GenericGuardrailAPI from litellm.types.llms.openai import ChatCompletionToolCallChunk +from litellm.responses.litellm_completion_transformation.compaction_items import compaction_summary from litellm.responses.litellm_completion_transformation.transformation import ( LiteLLMCompletionResponsesConfig, ) @@ -298,6 +299,32 @@ class TestOpenAIResponsesHandlerInputProcessing: assert result["input"][1]["content"] == "World [GUARDRAILED]" assert result["model"] == "gpt-4" + @pytest.mark.asyncio + async def test_compaction_summary_is_scanned_and_rewritten_inside_its_item(self) -> None: + affinity_prefix: Final = "litellm_enc:bW9kZWxfaWQ6ZGVwLWE=;" + data: Final = { + "input": [ + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": affinity_prefix + + json.dumps({"type": "compaction", "content": "Cat is Biscuit."}), + }, + {"role": "user", "content": "Cat name?", "type": "message"}, + ], + "model": "claude-sonnet-5-5", + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(data, MockGuardrail(guardrail_name="t")) + + rewritten: Final = result["input"][0]["encrypted_content"] + assert rewritten.startswith(affinity_prefix) + assert json.loads(rewritten.removeprefix(affinity_prefix)) == { + "type": "compaction", + "content": "Cat is Biscuit. [GUARDRAILED]", + } + assert result["input"][1]["content"] == "Cat name? [GUARDRAILED]" + @pytest.mark.asyncio async def test_process_input_list_with_multimodal_content(self): """Test processing list input with multimodal content""" @@ -3072,6 +3099,21 @@ class DictStructuredMessagesGuardrail(CustomGuardrail): return {**inputs, "structured_messages": {"evaluation": "allowed", "messages": []}} +class NameMaskingStructuredGuardrail(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: Optional[Any] = None, + ) -> GenericGuardrailAPIInputs: + rewritten: Final = [ + {**m, "content": m["content"].replace("Biscuit", "[NAME]")} if isinstance(m.get("content"), str) else m + for m in inputs.get("structured_messages") or [] + ] + return {**inputs, "structured_messages": rewritten} + + def _parallel_tool_call_input() -> list: return [ {"id": "fc_1", "type": "function_call", "call_id": "call_1", "name": "read_a", "arguments": "{}"}, @@ -3298,6 +3340,30 @@ class TestProvenancePatching: assistant_items = [item for item in result["input"] if item.get("role") == "assistant"] assert [_texts(item) for item in assistant_items] == [[COMPRESSED_MARKER]] + @pytest.mark.asyncio + async def test_masked_compaction_summary_stays_a_compaction_item_that_drops_older_history(self) -> None: + data: Final = { + "model": "claude-sonnet-5-5", + "input": [ + {"role": "user", "content": "My cat is Biscuit. " * 200}, + {"role": "assistant", "content": "Noted."}, + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": "Cat is Biscuit."}), + }, + {"role": "user", "content": "Cat name?"}, + ], + } + + result: Final = await OpenAIResponsesHandler().process_input_messages(data, NameMaskingStructuredGuardrail()) + + assert compaction_summary(result["input"][2]) == "Cat is [NAME]." + provider_messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=result["input"], responses_api_request={}, only_since_compaction=True + ) + assert [message["content"] for message in provider_messages] == ["Cat is [NAME].", "Cat name?"] + def test_provenance_bails_on_non_mapping_item(self): from litellm.llms.openai.responses.guardrail_translation.handler import _input_item_provenance @@ -4174,3 +4240,117 @@ class TestOpenAIResponsesHandlerStreamingScanKey: assert [inputs.get("texts") for inputs in guardrail.seen_inputs] == [["hi"]], guardrail.seen_inputs tool_calls = guardrail.seen_inputs[0].get("tool_calls") or [] assert [call["function"]["arguments"] for call in tool_calls] == ['{"city": "Paris"}'], tool_calls + + @pytest.mark.asyncio + async def test_a_tool_call_finished_mid_stream_reaches_the_guardrail_without_texts(self): + guardrail: Final = MockRecordingGuardrail(guardrail_name="test") + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=[self._finished_function_call(1, "fc_1", "Paris")[-1]], + guardrail_to_apply=guardrail, + request_data={}, + ) + assert [ + (sorted(inputs), [call["function"]["arguments"] for call in inputs.get("tool_calls") or []]) + for inputs in guardrail.seen_inputs + ] == [(["tool_calls"], ['{"city": "Paris"}'])] + + +class TestCompactionOutputSummaryReachesOutputGuardrails: + SUMMARY: Final = "The user lives at 1 Secret Lane." + + @classmethod + def _compaction_item(cls, summary: str | None = None) -> dict[str, str]: + return { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": summary or cls.SUMMARY}), + } + + @staticmethod + def _summary(item: object) -> str: + encrypted_content: Final = item["encrypted_content"] if isinstance(item, dict) else item.encrypted_content + return json.loads(encrypted_content)["content"] + + @staticmethod + def _lane_masking_guardrail() -> CustomGuardrail: + class MaskLane(CustomGuardrail): + async def apply_guardrail( + self, + inputs: GenericGuardrailAPIInputs, + request_data: dict, + input_type: Literal["request", "response"], + logging_obj: LiteLLMLoggingObj | None = None, + ) -> GenericGuardrailAPIInputs: + return { + **inputs, + "texts": [text.replace("Secret Lane", "[MASKED]") for text in inputs.get("texts", [])], + } + + return MaskLane(guardrail_name="mask-lane") + + @pytest.mark.asyncio + async def test_non_streaming_summary_is_masked_inside_its_item(self) -> None: + response: Final = ResponsesAPIResponse( + id="resp_1", + created_at=1, + model="claude-sonnet-5-5", + object="response", + status="completed", + output=[self._compaction_item()], + ) + + result: Final = await OpenAIResponsesHandler().process_output_response(response, self._lane_masking_guardrail()) + + assert self._summary(result.output[0]) == "The user lives at 1 [MASKED]." + + @pytest.mark.asyncio + async def test_ended_stream_summary_is_masked_in_the_done_item_and_the_envelope(self) -> None: + events: Final = [ + { + "type": "response.output_item.added", + "output_index": 0, + "item": {**self._compaction_item(), "encrypted_content": ""}, + }, + {"type": "response.output_item.done", "output_index": 0, "item": self._compaction_item()}, + { + "type": "response.completed", + "response": { + "id": "resp_1", + "created_at": 1, + "model": "claude-sonnet-5-5", + "output": [self._compaction_item()], + "status": "completed", + }, + }, + ] + + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=events, + guardrail_to_apply=self._lane_masking_guardrail(), + deliver_ended_stream_rewrites=True, + ) + + assert self._summary(events[1]["item"]) == "The user lives at 1 [MASKED]." + assert self._summary(events[2]["response"]["output"][0]) == "The user lives at 1 [MASKED]." + + @pytest.mark.asyncio + async def test_a_blocking_guardrail_rejects_a_finished_compaction_item_mid_stream(self) -> None: + with pytest.raises(HTTPException) as exc_info: + await OpenAIResponsesHandler().process_output_streaming_response( + responses_so_far=[ + {"type": "response.output_item.done", "output_index": 0, "item": self._compaction_item()} + ], + guardrail_to_apply=MockGuardrail(guardrail_name="block"), + ) + + assert exc_info.value.detail["texts"] == [self.SUMMARY] + + def test_a_changed_summary_changes_the_ended_stream_scan_key(self) -> None: + handler: Final = OpenAIResponsesHandler() + + def ended_key(summary: str) -> object: + return handler.get_streaming_scan_key( + [{"type": "response.completed", "response": {"output": [self._compaction_item(summary)]}}] + ) + + assert ended_key(self.SUMMARY) != ended_key("The user lives elsewhere.") diff --git a/tests/unit/proxy/guardrails/test_content_utils.py b/tests/unit/proxy/guardrails/test_content_utils.py index 920ffc77095..c14375135dc 100644 --- a/tests/unit/proxy/guardrails/test_content_utils.py +++ b/tests/unit/proxy/guardrails/test_content_utils.py @@ -1,5 +1,7 @@ """Tests for the shared guardrail content extraction helpers.""" +import json + from litellm.proxy.guardrails._content_utils import ( apply_redacted_messages_back, build_inspection_messages, @@ -167,6 +169,21 @@ def test_iter_message_text_inspects_reasoning_content_and_summary(): assert list(iter_message_text(data)) == ["content secret", "summary secret"] + +def test_iter_message_text_inspects_compaction_summary(): + data = { + "input": [ + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": "banned word"}), + }, + {"role": "user", "content": "next turn"}, + ] + } + assert list(iter_message_text(data)) == ["banned word", "next turn"] + + # ── walk_user_text ──────────────────────────────────────────────────────────── @@ -347,6 +364,30 @@ def test_walk_user_text_redacts_reasoning_content_and_summary(): assert item["id"] == "rs_1" + +def test_walk_user_text_redacts_compaction_summary_and_keeps_the_block(): + data = { + "input": [ + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps( + {"type": "compaction", "content": "AKIAEXAMPLE summary", "signature": "sig"} + ), + } + ] + } + visited = walk_user_text(data, lambda s: s.replace("AKIAEXAMPLE", "[REDACTED]")) + item = data["input"][0] + assert visited == 1 + assert (item["type"], item["id"]) == ("compaction", "cmp_1") + assert json.loads(item["encrypted_content"]) == { + "type": "compaction", + "content": "[REDACTED] summary", + "signature": "sig", + } + + # ── build_inspection_messages ───────────────────────────────────────────────── diff --git a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py index bd997f10322..76ad1e08e28 100644 --- a/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py +++ b/tests/unit/proxy/hooks/test_proxy_track_cost_callback.py @@ -415,6 +415,74 @@ async def test_async_post_call_failure_hook_does_not_clobber_guardrail_info_in_m assert metadata["standard_logging_guardrail_information"] == metadata_bucket_info +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_headers, caller_metadata, redaction_is_global, code_word_is_stored", + [ + ({"x-litellm-enable-message-redaction": "true"}, {"user_id": "claude-code-session"}, False, False), + ({"litellm-enable-message-redaction": "true"}, {}, False, False), + ({"user-agent": "claude-cli"}, {"user_id": "claude-code-session"}, False, True), + ({"user-agent": "claude-cli"}, {"headers": {"x-litellm-enable-message-redaction": "true"}}, False, False), + ({"litellm-disable-message-redaction": "true"}, {"user_id": "claude-code-session"}, True, False), + ], + ids=[ + "enable_header_redacts", + "legacy_enable_header_redacts", + "no_header_stores_prompt", + "caller_metadata_headers_still_apply", + "disable_header_never_loosens_global_redaction", + ], +) +async def test_async_post_call_failure_hook_honors_the_redaction_header_on_a_litellm_metadata_route( + monkeypatch: pytest.MonkeyPatch, + request_headers: dict[str, str], + caller_metadata: dict[str, object], + redaction_is_global: bool, + code_word_is_stored: bool, +): + monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true") + monkeypatch.setattr(litellm, "turn_off_message_logging", redaction_is_global) + writer: Final = MagicMock(spec=DBSpendUpdateWriter) + writer.update_database = AsyncMock() + logger: Final = ProxyDBLogger(spend_writer=lambda: writer) + request_input: Final = "The code word is VERMILION-42. Reply OK." + body_metadata: Final = dict(caller_metadata) + litellm_metadata_headers: Final = dict(request_headers) + + await logger.async_post_call_failure_hook( + request_data={ + "model": "gpt-5.6", + "input": request_input, + "metadata": body_metadata, + "litellm_metadata": {"headers": litellm_metadata_headers}, + "proxy_server_request": { + "url": "http://localhost/v1/responses", + "body": { + "model": "gpt-5.6", + "input": request_input, + "temperature": 7, + "metadata": body_metadata, + "litellm_metadata": {"headers": litellm_metadata_headers}, + }, + }, + }, + original_exception=Exception("temperature must be at most 2"), + user_api_key_dict=UserAPIKeyAuth(api_key="test_api_key"), + ) + + row: Final = get_logging_payload( + kwargs=writer.update_database.call_args.kwargs["kwargs"], + response_obj={}, + start_time=datetime.now(), + end_time=datetime.now(), + ) + stored_request: Final = json.loads(row["proxy_server_request"]) + assert stored_request["temperature"] == 7 + assert ("VERMILION-42" in row["proxy_server_request"]) is code_word_is_stored + assert stored_request["metadata"].get("headers") == caller_metadata.get("headers") + assert stored_request["litellm_metadata"]["headers"] == request_headers + + @pytest.mark.asyncio @pytest.mark.parametrize( "used_client_oauth_token, custom_llm_provider, expected", diff --git a/tests/unit/proxy/hooks/test_tpm_concurrent.py b/tests/unit/proxy/hooks/test_tpm_concurrent.py index bb2a89b1d98..3d1e494e36c 100644 --- a/tests/unit/proxy/hooks/test_tpm_concurrent.py +++ b/tests/unit/proxy/hooks/test_tpm_concurrent.py @@ -15,8 +15,9 @@ Redis. """ import asyncio +import json from datetime import datetime, timedelta -from typing import Any, Dict +from typing import Any, Dict, Final import pytest @@ -3727,3 +3728,50 @@ async def test_the_project_itpm_reservation_counts_the_request_off_the_event_loo if __name__ == "__main__": pytest.main([__file__, "-v", "-s"]) + + +_COMPACTION_SUMMARY: Final = "The user planned a Tokyo trip." +_NEXT_TURN: Final = {"type": "message", "role": "user", "content": "And Osaka?"} +_COMPACTED_INPUT: Final = [ + {"type": "message", "role": "user", "content": "Plan a Tokyo trip. " * 200}, + {"type": "message", "role": "assistant", "content": "Day one, Asakusa. " * 200}, + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": _COMPACTION_SUMMARY}), + }, + _NEXT_TURN, +] +_SUMMARY_INPUT: Final = [{"type": "message", "role": "user", "content": _COMPACTION_SUMMARY}, _NEXT_TURN] + + +def test_reservation_skips_the_history_a_compaction_item_replaces(rate_limiter): + handler, _cache = rate_limiter + compacted_request: Final = {"model": "gpt-4o", "input": _COMPACTED_INPUT, "max_output_tokens": 1} + + assert handler._estimate_input_and_output_tokens( + data=compacted_request, call_type="aresponses" + ) == handler._estimate_input_and_output_tokens( + data={**compacted_request, "input": _COMPACTED_INPUT[2:]}, call_type="aresponses" + ) + assert handler._estimate_precise_input_tokens( + compacted_request, model="gpt-4o", call_type="aresponses" + ) == handler._estimate_precise_input_tokens( + {**compacted_request, "input": _SUMMARY_INPUT}, model="gpt-4o", call_type="aresponses" + ) + + +def test_reservation_counts_a_compaction_item_by_its_summary(rate_limiter): + handler, _cache = rate_limiter + padded_item: Final = { + **_COMPACTED_INPUT[2], + "encrypted_content": json.dumps( + {"type": "compaction", "content": _COMPACTION_SUMMARY, "opaque_state": "x" * 4000} + ), + } + + assert handler._estimate_input_and_output_tokens( + data={"model": "gpt-4o", "input": [padded_item, _NEXT_TURN], "max_output_tokens": 1}, call_type="aresponses" + ) == handler._estimate_input_and_output_tokens( + data={"model": "gpt-4o", "input": _COMPACTED_INPUT[2:], "max_output_tokens": 1}, call_type="aresponses" + ) diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index abc1fe88c7a..985aee061bc 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -176,6 +176,105 @@ async def test_streaming_upstream_errors_keep_the_client_protocol( assert "error" in events[-1] +_COMPACTION_ITEM: Final = { + "type": "compaction", + "encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}), +} +_HISTORY_BEFORE_COMPACTION: Final = ( + {"role": "user", "content": "a long turn " * 200}, + {"role": "assistant", "content": "a long reply " * 200}, +) +_TURN_AFTER_COMPACTION: Final = {"role": "user", "content": "what is the cat's name?"} + + +async def _cursor_stream_failure_prompt_tokens( + monkeypatch: pytest.MonkeyPatch, request_input: list[Mapping[str, str]], fails_after_output: bool +) -> int: + import litellm.proxy.proxy_server as ps + from litellm.integrations.custom_logger import CustomLogger + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + + class _FailureRowRecorder(CustomLogger): + prompt_tokens: int | None = None + + async def async_post_call_failure_hook( + self, + request_data: dict, + original_exception: Exception, + user_api_key_dict: object, + traceback_str: str | None = None, + ) -> None: + usage: Final = request_data.get("combined_usage_object") + self.prompt_tokens = usage.prompt_tokens if isinstance(usage, litellm.Usage) else None + + model: Final = "gpt-6-astra" + upstream_url: Final = "https://streaming.example/v1" + created: Final = { + "type": "response.created", + "sequence_number": 0, + "response": { + "id": "resp_upstream", + "object": "response", + "created_at": 1, + "status": "in_progress", + "model": model, + "output": [], + }, + } + output_delta: Final = { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": "msg_partial", + "output_index": 0, + "content_index": 0, + "delta": "Biscuit", + } + failed: Final = {"type": "error", "error": {"message": "upstream broke", "code": "server_error", "type": None}} + events: Final = (created, output_delta, failed) if fails_after_output else (created, failed) + wire: Final = "".join("data: " + json.dumps(event) + "\n\n" for event in events) + recorder: Final = _FailureRowRecorder() + router: Final = litellm.Router( + model_list=[ + { + "model_name": model, + "litellm_params": {"model": "openai/" + model, "api_base": upstream_url, "api_key": "fixture-key"}, + } + ], + num_retries=0, + ) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(litellm, "callbacks", [recorder]) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setitem(app.dependency_overrides, user_api_key_auth, _auth_override) + with respx.mock as transport: + transport.post(upstream_url + "/responses").respond( + 200, content=wire, headers={"Content-Type": "text/event-stream"} + ) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app), base_url="http://testserver") as client: + result: Final = await client.post( + "/cursor/chat/completions", json={"model": model, "stream": True, "input": request_input} + ) + assert recorder.prompt_tokens is not None, result.text + return recorder.prompt_tokens + + +@pytest.mark.asyncio +@pytest.mark.parametrize("fails_after_output", [False, True]) +async def test_a_failed_cursor_stream_counts_its_input_from_the_newest_compaction_item( + monkeypatch: pytest.MonkeyPatch, fails_after_output: bool +) -> None: + """Cursor's route maps to no call type, yet it dispatches a Responses call, which sends + the provider only the newest compaction item onward, so the failure row counts that much.""" + resent: Final = await _cursor_stream_failure_prompt_tokens( + monkeypatch, [*_HISTORY_BEFORE_COMPACTION, _COMPACTION_ITEM, _TURN_AFTER_COMPACTION], fails_after_output + ) + dispatched_after_trim: Final = await _cursor_stream_failure_prompt_tokens( + monkeypatch, [_COMPACTION_ITEM, _TURN_AFTER_COMPACTION], fails_after_output + ) + + assert resent == dispatched_after_trim + + @pytest.mark.asyncio async def test_responses_api_background_polling_rejects_missing_input(): from fastapi import Response as FastAPIResponse @@ -4832,3 +4931,26 @@ def test_responses_routes_document_response_models_in_openapi_schema(): assert "output" in ok_200_properties("/v1/responses/{response_id}", "get") assert "deleted" in ok_200_properties("/v1/responses/{response_id}", "delete") assert "data" in ok_200_properties("/v1/responses/{response_id}/input_items", "get") + + +_COMPACTION_SUMMARY: Final = "The user planned a Tokyo trip." +_NEXT_TURN: Final = {"type": "message", "role": "user", "content": "And Osaka?"} +_COMPACTED_INPUT: Final = [ + {"type": "message", "role": "user", "content": "Plan a Tokyo trip. " * 200}, + {"type": "message", "role": "assistant", "content": "Day one, Asakusa. " * 200}, + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": _COMPACTION_SUMMARY}), + }, + _NEXT_TURN, +] +_SUMMARY_INPUT: Final = [{"type": "message", "role": "user", "content": _COMPACTION_SUMMARY}, _NEXT_TURN] + + +def test_input_token_count_reads_a_compaction_item_as_its_summary(): + from litellm.proxy.response_api_endpoints.endpoints import _responses_input_as_token_count_messages + + assert _responses_input_as_token_count_messages(_COMPACTED_INPUT, None) == ( + _responses_input_as_token_count_messages(_SUMMARY_INPUT, None) + ) diff --git a/tests/unit/proxy/spend_tracking/test_input_tokens.py b/tests/unit/proxy/spend_tracking/test_input_tokens.py index a1da6c79cd7..71f4dec2345 100644 --- a/tests/unit/proxy/spend_tracking/test_input_tokens.py +++ b/tests/unit/proxy/spend_tracking/test_input_tokens.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json from types import MappingProxyType from typing import Final @@ -27,3 +28,59 @@ async def test_large_input_is_still_counted() -> None: assert counts[CL100K_MODEL] == count_input_tokens_for_model(request_body=request_body, model=CL100K_MODEL) assert isinstance(counts, MappingProxyType) + + +_COMPACTION_ITEM: Final = { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": "Turn one planned Tokyo."}), +} +_TURN_AFTER_COMPACTION: Final = {"role": "user", "content": "And Osaka?"} + + +@pytest.mark.parametrize( + "earlier_item", + [ + pytest.param({"role": "user", "content": "Plan a Tokyo trip. " * 500}, id="user_turn"), + pytest.param({"role": ["system"], "content": "Plan a Tokyo trip."}, id="list_role"), + pytest.param({"role": {"name": "system"}, "content": "Plan a Tokyo trip."}, id="dict_role"), + ], +) +@pytest.mark.parametrize("model", [CL100K_MODEL, "claude-sonnet-5-5"]) +@pytest.mark.asyncio +async def test_history_before_a_compaction_item_is_not_counted(model: str, earlier_item: dict[str, object]) -> None: + compacted_body: Final = {"model": model, "input": [_COMPACTION_ITEM, _TURN_AFTER_COMPACTION]} + full_body: Final = {"model": model, "input": [earlier_item, _COMPACTION_ITEM, _TURN_AFTER_COMPACTION]} + + full_counts: Final = await count_input_tokens( + request_body=full_body, raw_body=json.dumps(full_body).encode(), models=(model,) + ) + compacted_counts: Final = await count_input_tokens( + request_body=compacted_body, raw_body=json.dumps(compacted_body).encode(), models=(model,) + ) + + assert full_counts[model] == compacted_counts[model] + assert count_input_tokens_for_model(request_body=full_body, model=model) == compacted_counts[model] + + +def _compaction_item_carrying(extra_block_fields: dict[str, str]) -> dict[str, str]: + return { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps( + {"type": "compaction", "content": "Turn one planned Tokyo.", **extra_block_fields} + ), + } + + +@pytest.mark.parametrize("model", [CL100K_MODEL, "claude-sonnet-5-5"]) +def test_a_compaction_item_counts_only_its_summary(model: str) -> None: + plain_body: Final = {"model": model, "input": [_compaction_item_carrying({}), _TURN_AFTER_COMPACTION]} + padded_body: Final = { + "model": model, + "input": [_compaction_item_carrying({"opaque_state": "x" * 4000}), _TURN_AFTER_COMPACTION], + } + + assert count_input_tokens_for_model(request_body=padded_body, model=model) == count_input_tokens_for_model( + request_body=plain_body, model=model + ) diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 55358a3b0de..78f5e1ace0c 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -14,8 +14,23 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm import litellm.constants as litellm_constants import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils -from litellm.constants import KUBERNETES_POD_ROUTING_KEY, LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_GENERATED_METADATA_KEY, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL -from litellm.constants import CALLER_SESSION_ID_METADATA_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, MANAGED_BACKGROUND_RESPONSE_METADATA_KEY +from litellm.constants import ( + KUBERNETES_POD_ROUTING_KEY, + LITELLM_TRUNCATED_PAYLOAD_FIELD, + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + LITTELM_CLI_SERVICE_ACCOUNT_NAME, + LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + MAX_SPEND_LOG_MODEL_NAME_LENGTH, + REDACTED_BY_LITELM_STRING, + SESSION_ID_GENERATED_METADATA_KEY, + SESSION_ID_OMITTED_METADATA_KEY, + UNKNOWN_MODEL_SPEND_LOG_MODEL, +) +from litellm.constants import ( + CALLER_SESSION_ID_METADATA_KEY, + INTERNAL_CALL_ORIGIN_METADATA_KEY, + MANAGED_BACKGROUND_RESPONSE_METADATA_KEY, +) from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps @@ -366,7 +381,10 @@ def test_caller_session_keeps_the_legacy_trace_id_with_correlation_logging_off(m def test_session_displaced_by_a_trace_header_wins_only_with_correlation_logging( monkeypatch: pytest.MonkeyPatch, correlation_in_logs: bool, expected_session_id: str ): - header_chain: Final = {"litellm_session_id": "trace-abc", "metadata": {"trace_id": "trace-abc", "session_id": "trace-abc"}} + header_chain: Final = { + "litellm_session_id": "trace-abc", + "metadata": {"trace_id": "trace-abc", "session_id": "trace-abc"}, + } session_id: Final = _chat_session_id( monkeypatch, header_chain, @@ -380,7 +398,11 @@ def test_session_displaced_by_a_trace_header_wins_only_with_correlation_logging( ("litellm_params", "metadata"), [ pytest.param({"litellm_session_id": "convo\x00x"}, {}, id="body-session"), - pytest.param({"litellm_session_id": "trace-abc"}, {CALLER_SESSION_ID_METADATA_KEY: "convo\x00x"}, id="beside-trace-header"), + pytest.param( + {"litellm_session_id": "trace-abc"}, + {CALLER_SESSION_ID_METADATA_KEY: "convo\x00x"}, + id="beside-trace-header", + ), ], ) def test_caller_session_postgres_cannot_store_keeps_the_trace_id( @@ -394,7 +416,10 @@ def test_caller_session_postgres_cannot_store_keeps_the_trace_id( [ pytest.param({"metadata": {}}, id="no-session"), pytest.param( - {"litellm_session_id": "minted-1", "metadata": {"session_id": "minted-1", SESSION_ID_GENERATED_METADATA_KEY: True}}, + { + "litellm_session_id": "minted-1", + "metadata": {"session_id": "minted-1", SESSION_ID_GENERATED_METADATA_KEY: True}, + }, id="minted-by-generate-policy", ), pytest.param({"litellm_session_id": "job-7", "metadata": {"trace_id": "job-7"}}, id="copied-from-trace-id"), @@ -732,6 +757,19 @@ def test_large_request_no_truncation_threshold(): assert "skipped" in truncated_content +@pytest.mark.parametrize( + ("max_string_length_prompt_in_db", "stored_tail"), + [(0, "claude-sonnet-5-5"), (1, "claude-sonnet-5-5"), (-1, "laude-sonnet-5-5"), (-2, "aude-sonnet-5-5")], +) +def test_a_limit_under_two_keeps_mains_tail_after_the_marker(max_string_length_prompt_in_db: int, stored_tail: str): + sanitized: Final = _sanitize_request_body_for_spend_logs_payload( + {"model": "claude-sonnet-5-5"}, max_string_length_prompt_in_db=max_string_length_prompt_in_db + ) + + assert sanitized["model"].startswith("...") + assert sanitized["model"].endswith(f") ...{stored_tail}") + + def test_small_request_no_truncation(): """ Test that small strings are not truncated by MAX_STRING_LENGTH_PROMPT_IN_DB @@ -750,9 +788,7 @@ def test_small_request_no_truncation(): # Verify the content was NOT truncated assert sanitized["messages"][0]["content"] == small_content - assert ( - len(sanitized["messages"][0]["content"]) == MAX_STRING_LENGTH_PROMPT_IN_DB - 100 - ) + assert len(sanitized["messages"][0]["content"]) == MAX_STRING_LENGTH_PROMPT_IN_DB - 100 def test_configurable_string_length_env_var(monkeypatch): @@ -1144,6 +1180,187 @@ def test_get_response_for_spend_logs_payload_truncates_large_base64(mock_should_ assert parsed["data"][0]["other_field"] == "value" +def test_get_response_for_spend_logs_payload_keeps_a_long_compaction_summary_whole(monkeypatch: pytest.MonkeyPatch): + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true") + filler = "detail " * (MAX_STRING_LENGTH_PROMPT_IN_DB // 7) + summary = f"{filler}The cat is named Biscuit. {filler}" + answer = f"{filler}OK{filler}" + payload = cast( + StandardLoggingPayload, + { + "response": { + "choices": [ + { + "message": { + "role": "assistant", + "content": answer, + "provider_specific_fields": { + "compaction_blocks": [{"type": "compaction", "content": summary}] + }, + } + } + ] + } + }, + ) + + message = json.loads(_get_response_for_spend_logs_payload(payload))["choices"][0]["message"] + assert message["provider_specific_fields"]["compaction_blocks"] == [{"type": "compaction", "content": summary}] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in message["content"] + + +def test_stored_request_keeps_only_the_summary_of_a_long_claude_compaction_input_item(monkeypatch: pytest.MonkeyPatch): + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true") + filler = "detail " * (MAX_STRING_LENGTH_PROMPT_IN_DB // 7) + summary = f"{filler}The cat is named Biscuit. {filler}" + claude_item = json.dumps({"type": "compaction", "content": summary, "padding": filler * 10}) + opaque_item = "gAAAA" + "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB * 2) + request_body = { + "input": [ + {"type": "compaction", "id": "cmp_1", "encrypted_content": claude_item}, + {"type": "compaction", "id": "cmp_2", "encrypted_content": opaque_item}, + {"role": "assistant", "content": [{"type": "compaction", "content": f"{filler}Biscuit{filler}"}]}, + {"role": "user", "content": f"{filler}What is my cat called?{filler}"}, + ], + "metadata": {"note": {"type": "compaction", "encrypted_content": claude_item}}, + } + + stored = json.loads( + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, litellm_params={"proxy_server_request": {"body": request_body}} + ) + ) + + claude_stored, opaque_stored, messages_block_stored, message_stored = stored["input"] + assert {**claude_stored, "encrypted_content": json.loads(claude_stored["encrypted_content"])} == { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": {"type": "compaction", "content": summary}, + } + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in opaque_stored["encrypted_content"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in messages_block_stored["content"][0]["content"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in message_stored["content"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored["metadata"]["note"]["encrypted_content"] + + +def _compaction_summary_past_the_summary_cap() -> str: + from litellm.constants import MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB + + filler: Final = "detail " * (MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB // 7) + return f"The cat is named Biscuit. {filler}The dog is named Rex." + + +def _assert_bounded_like_a_db_string(stored: str, summary: str) -> None: + from litellm.constants import MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB + + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in stored + assert len(stored) < MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB + 500 < len(summary) + 500 + assert stored.startswith("The cat is named Biscuit.") + assert stored.endswith("The dog is named Rex.") + + +@pytest.mark.parametrize("operator_cap_above_the_summary", [False, True]) +def test_spend_row_bounds_a_compaction_summary_at_the_larger_of_the_summary_cap_and_the_operator_cap( + monkeypatch: pytest.MonkeyPatch, operator_cap_above_the_summary: bool +): + monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true") + summary: Final = _compaction_summary_past_the_summary_cap() + if operator_cap_above_the_summary: + monkeypatch.setenv("MAX_STRING_LENGTH_PROMPT_IN_DB", str(len(summary) * 2)) + payload: Final = cast( + StandardLoggingPayload, + { + "response": { + "choices": [ + { + "message": { + "role": "assistant", + "content": "OK", + "provider_specific_fields": { + "compaction_blocks": [{"type": "compaction", "content": summary}] + }, + } + } + ] + } + }, + ) + request_body: Final = { + "input": [ + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": summary}), + } + ] + } + + response_message: Final = json.loads(_get_response_for_spend_logs_payload(payload))["choices"][0]["message"] + stored_item: Final = json.loads( + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, litellm_params={"proxy_server_request": {"body": request_body}} + ) + )["input"][0] + + stored_response_block: Final = response_message["provider_specific_fields"]["compaction_blocks"][0] + stored_request_block: Final = json.loads(stored_item["encrypted_content"]) + assert stored_request_block["type"] == stored_response_block["type"] == "compaction" + if operator_cap_above_the_summary: + assert stored_response_block["content"] == stored_request_block["content"] == summary + return + _assert_bounded_like_a_db_string(stored_response_block["content"], summary) + _assert_bounded_like_a_db_string(stored_request_block["content"], summary) + + +@pytest.mark.parametrize("summary_past_the_summary_cap", [False, True]) +def test_responses_shaped_spend_row_keeps_a_compaction_output_item_summary_bounded_like_a_chat_block( + monkeypatch: pytest.MonkeyPatch, summary_past_the_summary_cap: bool +): + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + from litellm.responses.litellm_completion_transformation.compaction_items import compaction_summary + from litellm.responses.utils import ResponsesAPIRequestUtils + + monkeypatch.setenv("STORE_PROMPTS_IN_SPEND_LOGS", "true") + filler: Final = "detail " * (MAX_STRING_LENGTH_PROMPT_IN_DB // 7) + summary: Final = ( + _compaction_summary_past_the_summary_cap() + if summary_past_the_summary_cap + else f"{filler}The cat is named Biscuit. {filler}" + ) + payload: Final = cast( + StandardLoggingPayload, + { + "response": { + "object": "response", + "output": [ + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + json.dumps({"type": "compaction", "content": summary}), "deployment-1" + ), + }, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "OK"}]}, + ], + } + }, + ) + + stored_item: Final = json.loads(_get_response_for_spend_logs_payload(payload))["output"][0] + model_id, _ = ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id(stored_item["encrypted_content"]) + stored_summary: Final = compaction_summary(stored_item) + assert model_id == "deployment-1" + assert stored_summary is not None + if summary_past_the_summary_cap: + _assert_bounded_like_a_db_string(stored_summary, summary) + return + assert stored_summary == summary + + @patch("litellm.proxy.spend_tracking.spend_tracking_utils.should_store_prompts_and_responses_in_spend_logs") def test_get_response_for_spend_logs_payload_strips_null_bytes(mock_should_store): """Regression for PostgreSQL 22P05: NUL bytes must be stripped from response.""" diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index a226ce8a4a9..92bf2b45fab 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -4066,6 +4066,72 @@ class TestHasAttributeErrorInChain: assert _has_attribute_error_in_chain(exc_a) is False +_COMPACTION_ITEM: Final = { + "type": "compaction", + "encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}), +} +_HISTORY_BEFORE_COMPACTION: Final = ( + {"role": "user", "content": "a long turn " * 200}, + {"role": "assistant", "content": "a long reply " * 200}, +) +_TURN_AFTER_COMPACTION: Final = {"role": "user", "content": "what is the cat's name?"} + + +class TestHandleLLMApiExceptionPricesWhatTheServerDispatched: + async def _failed_prompt_tokens( + self, + route_type: Literal["acompletion", "aresponses"], + logged_call_type: str, + request_input: list[dict[str, str]], + ) -> int: + processing_obj: Final = ProxyBaseLLMRequestProcessing(data={"model": "gpt-4o", "input": request_input}) + cursor_caller: Final = ProxyUserAPIKeyAuth(request_route="/cursor/chat/completions") + mock_request: Final = MagicMock(spec=Request) + mock_request.headers = {} + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging.alert_types = [] + _, logging_obj = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=cursor_caller, + proxy_logging_obj=proxy_logging, + proxy_config=MagicMock(spec=ProxyConfig), + route_type=route_type, + ) + logging_obj.model_call_details.update( + first_api_call_start_time=datetime.datetime.now(), + messages=request_input, + call_type=logged_call_type, + ) + with pytest.raises(ProxyException): + await processing_obj.handle_llm_api_exception( + e=Exception("boom"), + user_api_key_dict=cursor_caller, + proxy_logging_obj=proxy_logging, + ) + return processing_obj.data["combined_usage_object"].prompt_tokens + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "route_type, logged_call_type, trims_at_the_compaction", + [("aresponses", "aresponses", True), ("acompletion", "responses", False)], + ) + async def test_a_failed_call_counts_from_the_newest_compaction_only_when_dispatched_as_responses( + self, + route_type: Literal["acompletion", "aresponses"], + logged_call_type: str, + trims_at_the_compaction: bool, + ) -> None: + async def prompt_tokens(request_input: list[dict[str, str]]) -> int: + return await self._failed_prompt_tokens(route_type, logged_call_type, request_input) + + resent: Final = await prompt_tokens([*_HISTORY_BEFORE_COMPACTION, _COMPACTION_ITEM, _TURN_AFTER_COMPACTION]) + dispatched_after_trim: Final = await prompt_tokens([_COMPACTION_ITEM, _TURN_AFTER_COMPACTION]) + + assert (resent == dispatched_after_trim) is trims_at_the_compaction + assert resent >= dispatched_after_trim + + @pytest.mark.asyncio class TestHandleLLMApiExceptionDictDetail: """ @@ -10817,6 +10883,7 @@ class TestDetachedStreamFailureHook: "user_api_key_dict": user_api_key_dict, "original_exception": failure, "request_data": processing_obj.data, + "dispatched_call_type": "anthropic_messages", } ] diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 3f8e3bc93d1..d9def23bde8 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -1229,6 +1229,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "litellm_metadata": copy.deepcopy(malicious_metadata), "weights": {"gpt-3.5-turbo": {"forged-deployment-id": 100}}, "_router_weights": {"gpt-3.5-turbo": {"forged-deployment-id": 100}}, + "_encrypted_content_affinity_pinned": True, } updated = await add_litellm_data_to_request( @@ -1251,6 +1252,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): assert "_router_weights" not in updated assert "weights" not in updated["proxy_server_request"]["body"] assert "_router_weights" not in updated["proxy_server_request"]["body"] + assert "_encrypted_content_affinity_pinned" not in updated stripped_keys = { "disable_global_guardrails", diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index ab6a874a010..b5940e4c212 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -1,5 +1,6 @@ import asyncio import datetime as real_datetime +import json import smtplib import traceback from collections.abc import Mapping @@ -745,7 +746,9 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: post-dispatch failures keep zero spend. """ - async def _run(self, request_data, original_exception: Exception | None = None): + async def _run( + self, request_data, original_exception: Exception | None = None, request_route=None, dispatched_call_type=None + ): from unittest.mock import AsyncMock, patch from litellm.proxy._types import UserAPIKeyAuth @@ -756,7 +759,8 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: await proxy_logging_obj.post_call_failure_hook( request_data=request_data, original_exception=original_exception or Exception("boom"), - user_api_key_dict=UserAPIKeyAuth(), + user_api_key_dict=UserAPIKeyAuth(request_route=request_route), + dispatched_call_type=dispatched_call_type, ) def _timeout(self, model: str, cause: Exception) -> Exception: @@ -773,6 +777,38 @@ class TestPostCallFailureHookEstimatesDispatchedInputTokens: logging_obj.model_call_details = model_call_details return logging_obj + @pytest.mark.asyncio + @pytest.mark.parametrize( + "request_route, dispatched_call_type", + [("/v1/responses", None), ("/cursor/chat/completions", "aresponses")], + ) + async def test_a_failed_responses_request_counts_its_input_from_the_newest_compaction_item( + self, request_route, dispatched_call_type + ): + from datetime import datetime + + async def prompt_tokens(request_input): + request_data = { + "litellm_logging_obj": self._logging_obj( + { + "first_api_call_start_time": datetime.now(), + "model": FAILURE_USAGE_MODEL, + "messages": request_input, + "call_type": "aresponses", + } + ), + "metadata": {}, + } + await self._run(request_data, request_route=request_route, dispatched_call_type=dispatched_call_type) + return request_data["combined_usage_object"].prompt_tokens + + resent = await prompt_tokens( + [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION] + ) + + assert resent == await prompt_tokens([COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION]) + assert resent < await prompt_tokens([COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, TURN_AFTER_COMPACTION]) + @pytest.mark.asyncio async def test_timed_out_request_estimates_input_tokens_priced_at_input_rate(self): from datetime import datetime @@ -2164,29 +2200,29 @@ def test_a_system_prompt_reads_the_same_whatever_shape_it_arrived_in(system_inpu def test_a_system_prompt_is_counted_on_top_of_the_request(): - from litellm.proxy.utils import _count_request_input_tokens + from litellm.proxy.utils import count_request_input_tokens - without = _count_request_input_tokens(FAILURE_USAGE_MODEL, "hello world", None) - with_system = _count_request_input_tokens(FAILURE_USAGE_MODEL, "hello world", "be brief") + without = count_request_input_tokens(FAILURE_USAGE_MODEL, "hello world", None) + with_system = count_request_input_tokens(FAILURE_USAGE_MODEL, "hello world", "be brief") assert without > 0 assert with_system > without def test_a_request_with_nothing_in_it_counts_zero(): - from litellm.proxy.utils import _count_request_input_tokens + from litellm.proxy.utils import count_request_input_tokens - assert _count_request_input_tokens(FAILURE_USAGE_MODEL, [], None) == 0 - assert _count_request_input_tokens(FAILURE_USAGE_MODEL, None, None) == 0 + assert count_request_input_tokens(FAILURE_USAGE_MODEL, [], None) == 0 + assert count_request_input_tokens(FAILURE_USAGE_MODEL, None, None) == 0 def test_a_failed_dispatch_is_estimated_as_input_only(): - from litellm.proxy.utils import _count_request_input_tokens, _estimate_dispatched_failure_usage + from litellm.proxy.utils import count_request_input_tokens, _estimate_dispatched_failure_usage usage = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None) assert usage is not None - assert usage.prompt_tokens == _count_request_input_tokens(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None) + assert usage.prompt_tokens == count_request_input_tokens(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None) assert usage.completion_tokens == 0 assert usage.total_tokens == usage.prompt_tokens @@ -2450,7 +2486,7 @@ def test_earlier_attempts_are_priced_from_the_prompt_estimate_not_the_final_stre lifted = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}, provider_billed=True) - estimate = _estimated_prompt_usage(logging_obj.model_call_details, {}) + estimate = _estimated_prompt_usage(logging_obj.model_call_details, {}, None) assert lifted["combined_usage_object"] is recovered assert [(a.usage.prompt_tokens, a.usage.completion_tokens) for a in lifted["billed_failed_attempts"]] == [ (estimate.prompt_tokens, 0) @@ -3336,6 +3372,83 @@ def test_a_request_that_timed_out_at_a_provider_bills_its_input_at_the_input_rat assert response_cost == pytest.approx(expected_input_cost) + +COMPACTION_INSTRUCTIONS = {"role": "system", "content": "answer briefly"} +COMPACTED_HISTORY = [ + {"role": "user", "content": "a long turn " * 200}, + {"role": "assistant", "content": "a long reply " * 200}, +] +COMPACTION_ITEM = { + "type": "compaction", + "encrypted_content": json.dumps({"type": "compaction", "content": "the cat is named Biscuit"}), +} +TURN_AFTER_COMPACTION = {"role": "user", "content": "what is the cat's name?"} + + +def _failed_call_prompt_tokens(call_type, dispatched_call_type, request_input): + from litellm.proxy.utils import _failure_usage_to_lift + + lifted = _failure_usage_to_lift( + model_call_details={"call_type": call_type, "model": FAILURE_USAGE_MODEL, "messages": request_input}, + request_body={}, + dispatched=True, + dispatched_call_type=dispatched_call_type, + ) + assert lifted is not None + return lifted[0].prompt_tokens + + +@pytest.mark.parametrize( + "call_type, dispatched_call_type", + [ + ("acompletion", "acompletion"), + ("aanthropic_messages", "anthropic_messages"), + ("responses", "acompletion"), + ], +) +def test_a_failed_call_off_the_responses_route_counts_the_history_before_a_compaction_typed_message( + call_type, dispatched_call_type +): + def prompt_tokens(request_input): + return _failed_call_prompt_tokens(call_type, dispatched_call_type, request_input) + + sent = prompt_tokens([COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION]) + + assert sent > prompt_tokens([COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION]) + + +def test_an_earlier_timed_out_responses_attempt_counts_its_input_from_the_newest_compaction_item(): + from litellm.litellm_core_utils.get_litellm_params import get_litellm_params + from litellm.proxy.utils import _failure_fields_to_lift + + def earlier_attempt_prompt_tokens(request_input): + logging_obj = _dispatched_failure_logging_obj( + FAILURE_USAGE_MODEL, + get_litellm_params( + metadata={ + "attempted_retries": 1, + "model_info": {"id": "dep-compaction-final"}, + "previous_models": ( + {"deployment_id": "dep-compaction-final", "exception_type": "Timeout", "attempted_retries": 0}, + ), + } + ), + ) + logging_obj.model_call_details["call_type"] = "aresponses" + logging_obj.model_call_details["messages"] = request_input + lifted = _failure_fields_to_lift( + {"litellm_logging_obj": logging_obj}, provider_billed=True, dispatched_call_type="aresponses" + ) + (earlier_attempt,) = lifted["billed_failed_attempts"] + return earlier_attempt.usage.prompt_tokens + + sent = earlier_attempt_prompt_tokens( + [COMPACTION_INSTRUCTIONS, *COMPACTED_HISTORY, COMPACTION_ITEM, TURN_AFTER_COMPACTION] + ) + + assert sent == earlier_attempt_prompt_tokens([COMPACTION_INSTRUCTIONS, COMPACTION_ITEM, TURN_AFTER_COMPACTION]) + + @pytest.mark.parametrize( "current_attempt_api_call_start_time", [None, 1700000001.0], diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 94efef050a6..36df738c0bb 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -1,15 +1,17 @@ import json from collections.abc import Mapping, Sequence from itertools import chain -from typing import Final +from typing import Final, cast from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx +from openai.types.responses.response_compaction_item import ResponseCompactionItem from pydantic import BaseModel import litellm +from litellm import CustomLLM from litellm.constants import REDACTED_BY_LITELLM from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.responses.litellm_completion_transformation.handler import ( @@ -52,6 +54,7 @@ async def test_async_response_api_handler_merges_trace_id_without_error(): instructions: str | None = None, request_kwargs: Mapping[str, object] | None = None, previous_response_client_ids: tuple[str, ...] = (), + input_replaces_history: bool = False, ) -> dict[str, object]: litellm_completion_request["litellm_trace_id"] = "session-trace" return litellm_completion_request @@ -561,3 +564,637 @@ def test_streamed_responses_over_anthropic_emit_the_responses_event_sequence(res assert deltas == "Argentina won." assert completed.response.status == "completed" assert completed.response.output[0].content[0].text == "Argentina won." + + +_ANTHROPIC_MESSAGES_URL: Final = "https://api.anthropic.com/v1/messages" +_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "The user pasted a long document and asked about it."} +_FIRST_INPUT: Final = ({"role": "user", "content": "What is the first word of the document?"},) +_NEXT_INPUT: Final = ({"role": "user", "content": "And the last word?"},) +_COMPACT_AT_50K: Final = ({"type": "compaction", "compact_threshold": 50000},) + + +def _anthropic_message(*content: Mapping[str, object]) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "msg_compacted", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": list(content), + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + ) + + +def _replay(output: Sequence[BaseModel]) -> list[object]: + return [*_FIRST_INPUT, *(item.model_dump(exclude_none=True) for item in output), *_NEXT_INPUT] + + +def _sent_assistant_content(route: respx.Route) -> list[object]: + sent: Final = json.loads(route.calls.last.request.content) + return next(message["content"] for message in sent["messages"] if message["role"] == "assistant") + + +def test_responses_over_anthropic_return_the_compaction_block_and_send_it_back( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + side_effect=[ + _anthropic_message(_COMPACTION_BLOCK, {"type": "text", "text": "The."}), + _anthropic_message({"type": "text", "text": "dog."}), + ] + ) + + first: Final = litellm.responses( + model=_MODEL, input=list(_FIRST_INPUT), context_management=list(_COMPACT_AT_50K), api_key="mock_api_key" + ) + litellm.responses( + model=_MODEL, input=_replay(first.output), context_management=list(_COMPACT_AT_50K), api_key="mock_api_key" + ) + + assert [item.type for item in first.output] == ["compaction", "message"] + assert _sent_assistant_content(route) == [_COMPACTION_BLOCK, {"type": "text", "text": "The."}] + + +def test_streamed_responses_over_anthropic_emit_the_compaction_item_and_send_it_back( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + side_effect=[ + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_anthropic_sse( + { + "type": "message_start", + "message": { + "id": "msg_compacted_stream", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "usage": {"input_tokens": 9, "output_tokens": 1}, + }, + }, + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "compaction", "content": None}, + }, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "compaction_delta", "content": _COMPACTION_BLOCK["content"]}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 1, "delta": {"type": "text_delta", "text": "The."}}, + {"type": "content_block_stop", "index": 1}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}}, + {"type": "message_stop"}, + ), + ), + _anthropic_message({"type": "text", "text": "dog."}), + ] + ) + + events: Final = tuple( + litellm.responses( + model=_MODEL, + input=list(_FIRST_INPUT), + context_management=list(_COMPACT_AT_50K), + stream=True, + api_key="mock_api_key", + ) + ) + item_events: Final = tuple( + event + for event in events + if event.type in (ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE) + ) + completed: Final = events[-1].response + litellm.responses( + model=_MODEL, input=_replay(completed.output), context_management=list(_COMPACT_AT_50K), api_key="mock_api_key" + ) + + assert [(event.type, event.output_index, event.item.type) for event in item_events] == [ + (ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, 0, "compaction"), + (ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, 1, "message"), + (ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, 0, "compaction"), + (ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, 1, "message"), + ] + assert item_events[0].item.encrypted_content == "" + assert item_events[0].item.id == item_events[2].item.id + assert [(item.type, getattr(item, "encrypted_content", None)) for item in completed.output[:1]] == [ + ("compaction", item_events[2].item.encrypted_content) + ] + assert completed.output[0].id == item_events[2].item.id + assert _sent_assistant_content(route) == [_COMPACTION_BLOCK, {"type": "text", "text": "The."}] + + +def test_streamed_responses_over_anthropic_answer_after_a_failed_compaction(respx_mock: respx.MockRouter) -> None: + respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_anthropic_sse( + { + "type": "message_start", + "message": { + "id": "msg_failed_compaction", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "usage": {"input_tokens": 9, "output_tokens": 1}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "compaction", "content": None}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "compaction_delta", "content": None}}, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 1, "delta": {"type": "text_delta", "text": "The."}}, + {"type": "content_block_stop", "index": 1}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}}, + {"type": "message_stop"}, + ), + ) + ) + + events: Final = tuple( + litellm.responses( + model=_MODEL, + input=list(_FIRST_INPUT), + context_management=list(_COMPACT_AT_50K), + stream=True, + api_key="mock_api_key", + ) + ) + + assert events[-1].type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + assert [item.type for item in events[-1].response.output] == ["message"] + assert events[-1].response.output[0].content[0].text == "The." + + +@pytest.mark.asyncio +async def test_async_streamed_compaction_item_is_done_before_the_reasoning_that_follows_it() -> None: + anthropic: Final = httpx.MockTransport( + lambda _request: httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_anthropic_sse( + { + "type": "message_start", + "message": { + "id": "msg_compacted_thinking", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "usage": {"input_tokens": 9, "output_tokens": 1}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "compaction", "content": None}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "compaction_delta", "content": _COMPACTION_BLOCK["content"]}, + }, + {"type": "content_block_stop", "index": 0}, + {"type": "content_block_start", "index": 1, "content_block": {"type": "thinking", "thinking": ""}}, + {"type": "content_block_delta", "index": 1, "delta": {"type": "thinking_delta", "thinking": "Recall."}}, + {"type": "content_block_delta", "index": 1, "delta": {"type": "signature_delta", "signature": "sig"}}, + {"type": "content_block_stop", "index": 1}, + {"type": "content_block_start", "index": 2, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 2, "delta": {"type": "text_delta", "text": "The."}}, + {"type": "content_block_stop", "index": 2}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}}, + {"type": "message_stop"}, + ), + ) + ) + + stream: Final = await litellm.aresponses( + model=_MODEL, + input=list(_FIRST_INPUT), + context_management=list(_COMPACT_AT_50K), + stream=True, + api_key="mock_api_key", + client=AsyncHTTPHandler(transport=anthropic), + ) + events: Final = [event async for event in stream] + done_items: Final = [ + (event.output_index, event.item.type) + for event in events + if event.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE + ] + reasoning_done: Final = next( + index + for index, event in enumerate(events) + if event.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE and event.item.type == "reasoning" + ) + sequence_numbers: Final = [ + event.sequence_number for event in events[: reasoning_done + 1] if getattr(event, "sequence_number", None) + ] + + assert done_items == [(0, "compaction"), (1, "reasoning"), (2, "message")] + assert [item.type for item in events[-1].response.output] == ["compaction", "reasoning", "message"] + assert sequence_numbers == sorted(set(sequence_numbers)) + + +def _compaction_stream_events(*summaries: str) -> tuple[Mapping[str, object], ...]: + return tuple( + chain.from_iterable( + ( + { + "type": "content_block_start", + "index": 2 * index, + "content_block": {"type": "compaction", "content": None}, + }, + { + "type": "content_block_delta", + "index": 2 * index, + "delta": {"type": "compaction_delta", "content": summary}, + }, + {"type": "content_block_stop", "index": 2 * index}, + {"type": "content_block_start", "index": 2 * index + 1, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 2 * index + 1, "delta": {"type": "text_delta", "text": "Ok."}}, + {"type": "content_block_stop", "index": 2 * index + 1}, + ) + for index, summary in enumerate(summaries) + ) + ) + + +def test_streamed_compaction_item_done_carries_the_same_newest_block_as_the_completed_response( + respx_mock: respx.MockRouter, +) -> None: + respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + content=_anthropic_sse( + { + "type": "message_start", + "message": { + "id": "msg_compacted_twice", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [], + "usage": {"input_tokens": 9, "output_tokens": 1}, + }, + }, + *_compaction_stream_events("First summary.", "Second summary."), + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 4}}, + {"type": "message_stop"}, + ), + ) + ) + + events: Final = tuple( + litellm.responses( + model=_MODEL, + input=list(_FIRST_INPUT), + context_management=list(_COMPACT_AT_50K), + stream=True, + api_key="mock_api_key", + ) + ) + streamed: Final = tuple( + (event.item.id, json.loads(event.item.encrypted_content)["content"]) + for event in events + if event.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE and event.item.type == "compaction" + ) + completed: Final = tuple( + (item.id, json.loads(item.encrypted_content)["content"]) + for item in events[-1].response.output + if item.type == "compaction" + ) + + assert streamed == completed + assert [summary for _, summary in completed] == ["Second summary."] + + +def test_responses_over_anthropic_send_back_a_compaction_item_replayed_as_sdk_output( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + side_effect=[ + _anthropic_message(_COMPACTION_BLOCK, {"type": "text", "text": "The."}), + _anthropic_message({"type": "text", "text": "dog."}), + ] + ) + + first: Final = litellm.responses( + model=_MODEL, input=list(_FIRST_INPUT), context_management=list(_COMPACT_AT_50K), api_key="mock_api_key" + ) + litellm.responses( + model=_MODEL, + input=[*_FIRST_INPUT, *first.output, *_NEXT_INPUT], + context_management=list(_COMPACT_AT_50K), + api_key="mock_api_key", + ) + + assert _sent_assistant_content(route)[0] == _COMPACTION_BLOCK + + +@pytest.mark.asyncio +async def test_router_with_encrypted_content_affinity_sends_the_compaction_block_back( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + side_effect=[ + _anthropic_message(_COMPACTION_BLOCK, {"type": "text", "text": "The."}), + _anthropic_message({"type": "text", "text": "dog."}), + ] + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "claude", + "litellm_params": {"model": _MODEL, "api_key": "mock_api_key"}, + "model_info": {"id": "deployment-a"}, + } + ], + optional_pre_call_checks=["encrypted_content_affinity"], + ) + + first: Final = await router.aresponses( + model="claude", input=list(_FIRST_INPUT), context_management=list(_COMPACT_AT_50K) + ) + await router.aresponses(model="claude", input=_replay(first.output), context_management=list(_COMPACT_AT_50K)) + + assert first.output[0].encrypted_content.startswith("litellm_enc:") + assert _sent_assistant_content(route) == [_COMPACTION_BLOCK, {"type": "text", "text": "The."}] + assert [key for key in json.loads(route.calls.last.request.content) if key.startswith("_")] == [] + + +_DOCUMENT_BEFORE_COMPACTION: Final = "The design document pasted before the compaction." +_TURN_BEFORE_COMPACTION: Final = { + "request_id": "chatcmpl-before-compaction", + "call_type": "aresponses", + "session_id": "session-compacted", + "proxy_server_request": { + "model": _MODEL, + "input": [ + {"role": "developer", "content": "Use metric units."}, + {"role": "user", "content": _DOCUMENT_BEFORE_COMPACTION}, + ], + }, + "response": { + "id": "chatcmpl-before-compaction", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "Read it."}}], + }, +} + + +class _RecordingRawAnthropicMessages: + def __init__(self) -> None: + self.body: str = "" + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.body = request.content.decode() + return _anthropic_message({"type": "text", "text": "dog."}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("context_management", [None, list(_COMPACT_AT_50K)], ids=["summary_text", "native_block"]) +@pytest.mark.parametrize( + "compaction_item", + [ + pytest.param( + {"type": "compaction", "id": "cmp_caller", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + id="param_dict", + ), + pytest.param( + ResponseCompactionItem(type="compaction", id="cmp_caller", encrypted_content=json.dumps(_COMPACTION_BLOCK)), + id="sdk_output_item", + ), + ], +) +async def test_previous_response_id_with_a_compaction_item_in_input_replays_only_the_stored_instructions( + context_management: list[Mapping[str, object]] | None, + compaction_item: Mapping[str, object] | ResponseCompactionItem, +) -> None: + anthropic: Final = _RecordingRawAnthropicMessages() + + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_TURN_BEFORE_COMPACTION])): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-before-compaction", + input=[compaction_item, *_NEXT_INPUT], + context_management=context_management, + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + assert _DOCUMENT_BEFORE_COMPACTION not in anthropic.body + assert "Read it." not in anthropic.body + assert "Use metric units." in anthropic.body + assert _COMPACTION_BLOCK["content"] in anthropic.body + assert _NEXT_INPUT[0]["content"] in anthropic.body + + +class _RecordingCustomLLM(CustomLLM): + def __init__(self) -> None: + super().__init__() + self.sent_messages: list[object] = [] + + def completion(self, *args: object, **kwargs: object) -> ModelResponse: + self.sent_messages.extend(cast(list[object], kwargs["messages"])) + return litellm.completion( + model="gpt-5.6", messages=[{"role": "user", "content": "Hi."}], mock_response="dog." + ) + + +def test_responses_over_a_custom_provider_send_a_compaction_item_as_summary_text( + monkeypatch: pytest.MonkeyPatch, +) -> None: + custom_llm: Final = _RecordingCustomLLM() + monkeypatch.setattr(litellm, "provider_list", list(litellm.provider_list)) + monkeypatch.setattr(litellm, "_custom_providers", list(litellm._custom_providers)) + monkeypatch.setattr( + litellm, "custom_provider_map", [{"provider": "compaction-test-llm", "custom_handler": custom_llm}] + ) + compaction_item: Final = { + "type": "compaction", + "id": "cmp_earlier", + "encrypted_content": json.dumps(_COMPACTION_BLOCK), + } + + litellm.responses( + model="compaction-test-llm/my-model", + input=[*_FIRST_INPUT, {"role": "assistant", "content": "The."}, compaction_item, *_NEXT_INPUT], + context_management=list(_COMPACT_AT_50K), + drop_params=True, + ) + + assert custom_llm.sent_messages == [ + {"role": "user", "content": _COMPACTION_BLOCK["content"]}, + *_NEXT_INPUT, + ] + + +_WEATHER_CALL: Final = { + "id": "toolu_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, +} +_TOOL_TURN_BEFORE_COMPACTION: Final = { + **_TURN_BEFORE_COMPACTION, + "request_id": "chatcmpl-tool-turn", + "response": { + "id": "chatcmpl-tool-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "content": None, "tool_calls": [_WEATHER_CALL]}, + } + ], + }, +} + + +def _content_blocks(message: Mapping[str, object]) -> list[Mapping[str, object]]: + content: Final = message["content"] + return content if isinstance(content, list) else [{"type": "text", "text": content}] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("context_management", [None, list(_COMPACT_AT_50K)], ids=["summary_text", "native_block"]) +async def test_previous_response_id_with_a_compaction_item_keeps_the_stored_call_its_tool_output_answers( + context_management: list[Mapping[str, object]] | None, +) -> None: + anthropic: Final = _RecordingRawAnthropicMessages() + compaction_item: Final = { + "type": "compaction", + "id": "cmp_caller", + "encrypted_content": json.dumps(_COMPACTION_BLOCK), + } + tool_output: Final = {"type": "function_call_output", "call_id": "toolu_weather", "output": "18C and sunny"} + + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_TOOL_TURN_BEFORE_COMPACTION])): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-tool-turn", + input=[compaction_item, tool_output], + context_management=context_management, + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + messages: Final = json.loads(anthropic.body)["messages"] + call_turn: Final = next( + index + for index, message in enumerate(messages) + if any(block["type"] == "tool_use" for block in _content_blocks(message)) + ) + tool_use: Final = next(block for block in _content_blocks(messages[call_turn]) if block["type"] == "tool_use") + assert (tool_use["id"], tool_use["input"]) == ("toolu_weather", {"city": "Paris"}) + assert _content_blocks(messages[call_turn + 1])[0]["type"] == "tool_result" + assert _content_blocks(messages[call_turn + 1])[0]["tool_use_id"] == "toolu_weather" + assert _COMPACTION_BLOCK["content"] in json.dumps(messages[: call_turn + 1]) + assert _DOCUMENT_BEFORE_COMPACTION not in anthropic.body + + +_OSAKA_CALL: Final = {**_WEATHER_CALL, "function": {"name": "get_weather", "arguments": '{"city": "Osaka"}'}} +_TOOL_TURN_REUSING_THE_CALL_ID: Final = { + **_TOOL_TURN_BEFORE_COMPACTION, + "request_id": "chatcmpl-osaka-turn", + "proxy_server_request": { + "model": _MODEL, + "input": [{"type": "function_call_output", "call_id": "toolu_weather", "output": "18C and sunny"}], + }, + "response": { + "id": "chatcmpl-osaka-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": {"role": "assistant", "content": None, "tool_calls": [_OSAKA_CALL]}, + } + ], + }, +} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("context_management", [None, list(_COMPACT_AT_50K)], ids=["summary_text", "native_block"]) +async def test_previous_response_id_with_a_compaction_item_restores_only_the_newest_stored_call_for_a_reused_call_id( + context_management: list[Mapping[str, object]] | None, +) -> None: + anthropic: Final = _RecordingRawAnthropicMessages() + compaction_item: Final = { + "type": "compaction", + "id": "cmp_caller", + "encrypted_content": json.dumps(_COMPACTION_BLOCK), + } + tool_output: Final = {"type": "function_call_output", "call_id": "toolu_weather", "output": "12C and rain"} + spend_logs: Final = [_TOOL_TURN_BEFORE_COMPACTION, _TOOL_TURN_REUSING_THE_CALL_ID] + + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient(spend_logs)): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-osaka-turn", + input=[compaction_item, tool_output], + context_management=context_management, + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + messages: Final = json.loads(anthropic.body)["messages"] + blocks: Final = list( + chain.from_iterable( + ((index, block) for block in _content_blocks(message)) for index, message in enumerate(messages) + ) + ) + tool_uses: Final = [(index, block["id"], block["input"]) for index, block in blocks if block["type"] == "tool_use"] + tool_results: Final = [(index, block["tool_use_id"]) for index, block in blocks if block["type"] == "tool_result"] + assert [(tool_id, tool_input) for _, tool_id, tool_input in tool_uses] == [("toolu_weather", {"city": "Osaka"})] + assert tool_results == [(tool_uses[0][0] + 1, "toolu_weather")] + assert "12C and rain" in json.dumps(messages[tool_uses[0][0] + 1]) + + +def test_responses_over_anthropic_send_a_compaction_item_as_summary_text_when_the_request_does_not_compact( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post(_ANTHROPIC_MESSAGES_URL).mock( + return_value=_anthropic_message({"type": "text", "text": "dog."}) + ) + compaction_item: Final = { + "type": "compaction", + "id": "cmp_earlier", + "encrypted_content": json.dumps(_COMPACTION_BLOCK), + } + + litellm.responses( + model=_MODEL, + input=[*_FIRST_INPUT, {"role": "assistant", "content": "The."}, compaction_item, *_NEXT_INPUT], + api_key="mock_api_key", + ) + + sent: Final = json.loads(route.calls.last.request.content) + assert sent["messages"] == [ + { + "role": "user", + "content": [ + {"type": "text", "text": _COMPACTION_BLOCK["content"]}, + {"type": "text", "text": _NEXT_INPUT[0]["content"]}, + ], + } + ] diff --git a/tests/unit/responses/litellm_completion_transformation/test_compaction_items.py b/tests/unit/responses/litellm_completion_transformation/test_compaction_items.py new file mode 100644 index 00000000000..e0ef3fbb552 --- /dev/null +++ b/tests/unit/responses/litellm_completion_transformation/test_compaction_items.py @@ -0,0 +1,87 @@ +import json +from typing import Final + +import pytest + +from litellm.responses.litellm_completion_transformation.compaction_items import ( + compaction_output_item, + holds_compaction_summary, + response_holds_compaction_block, + with_compaction_summary, +) +from litellm.responses.utils import ResponsesAPIRequestUtils + +_BLOCK: Final = {"type": "compaction", "content": "Turn one planned Tokyo."} +_ITEM: Final = {"type": "compaction", "id": "cmp_1", "encrypted_content": json.dumps(_BLOCK)} + + +@pytest.mark.parametrize( + "newer_entry", + [ + pytest.param("not a block", id="not_a_mapping"), + pytest.param({"type": "compaction", "content": None}, id="summary_not_streamed_yet"), + pytest.param({"type": "compaction", "content": ""}, id="empty_summary"), + pytest.param({"type": "text", "content": "Hi."}, id="not_a_compaction"), + ], +) +def test_compaction_output_item_carries_the_newest_block_that_holds_a_summary(newer_entry: object) -> None: + item: Final = compaction_output_item({"compaction_blocks": [_BLOCK, newer_entry]}) + + assert item is not None + assert json.loads(item.encrypted_content) == _BLOCK + + +@pytest.mark.parametrize( + ("response", "holds"), + [ + pytest.param( + {"choices": [{"message": {"role": "assistant", "provider_specific_fields": {"compaction_blocks": [_BLOCK]}}}]}, + True, + id="chat_answer_compacted", + ), + pytest.param({"choices": [{"message": {"role": "assistant", "content": "Hi."}}]}, False, id="chat_answer"), + pytest.param({"object": "response", "output": [_ITEM]}, True, id="responses_output_compacted"), + pytest.param( + {"object": "response", "output": [{"type": "compaction", "id": "cmp_openai", "encrypted_content": "gAAAA"}]}, + False, + id="responses_output_with_an_opaque_compaction", + ), + pytest.param({"choices": "not a list"}, False, id="malformed_choices"), + pytest.param(None, False, id="no_stored_response"), + ], +) +def test_response_holds_compaction_block_reads_chat_and_responses_shaped_rows(response: object, holds: bool) -> None: + assert response_holds_compaction_block(response) is holds + + +@pytest.mark.parametrize( + ("encrypted_content", "holds"), + [ + pytest.param(json.dumps(_BLOCK), True, id="litellm_block"), + pytest.param( + ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(json.dumps(_BLOCK), "deployment-1"), + True, + id="affinity_wrapped_block", + ), + pytest.param("gAAAAABopaque", False, id="openai_token"), + pytest.param(None, False, id="missing"), + ], +) +def test_only_a_compaction_item_litellm_can_decode_holds_a_summary(encrypted_content: object, holds: bool) -> None: + assert holds_compaction_summary([{"type": "compaction", "id": "cmp_1", "encrypted_content": encrypted_content}]) is ( + holds + ) + + +def test_with_compaction_summary_rewrites_the_summary_and_keeps_the_affinity_wrap() -> None: + wrapped: Final = ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id(json.dumps(_BLOCK), "deployment-1") + + rewritten: Final = with_compaction_summary(wrapped, "[REDACTED]") + model_id, original = ResponsesAPIRequestUtils.unwrap_encrypted_content_with_model_id(rewritten) + + assert model_id == "deployment-1" + assert json.loads(original) == {**_BLOCK, "content": "[REDACTED]"} + + +def test_with_compaction_summary_leaves_a_token_it_cannot_read_unchanged() -> None: + assert with_compaction_summary("gAAAAABopaque", "[REDACTED]") == "gAAAAABopaque" diff --git a/tests/unit/responses/litellm_completion_transformation/test_count_request.py b/tests/unit/responses/litellm_completion_transformation/test_count_request.py index 464bdb1a367..0477ae15b8e 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_count_request.py +++ b/tests/unit/responses/litellm_completion_transformation/test_count_request.py @@ -10,6 +10,7 @@ from litellm.responses.litellm_completion_transformation.count_request import ( ChatCountRequest, chat_count_request, ) +from litellm.responses.litellm_completion_transformation.transformation import ChatCompletionSession _MARK: Final[dict[str, JsonValue]] = {"type": "ephemeral"} _MARKED_RESPONSES_REQUEST: Final[dict[str, JsonValue]] = { @@ -125,3 +126,77 @@ async def test_claude_count_request_leaves_out_tools_the_deployment_drops() -> N assert isinstance(dropped, ChatCountRequest) assert dropped.tools is None assert dropped == await _claude_count({"input": [_QUESTION]}, policy) + + +_SUMMARY: Final = "Earlier turns covered the Tokyo trip." +_COMPACTED_INPUT: Final[list[JsonValue]] = [ + {"type": "compaction", "id": "cmp_1", "encrypted_content": json.dumps({"type": "compaction", "content": _SUMMARY})}, + {"type": "message", "role": "user", "content": "And Osaka?"}, +] +_COMPACTION_REQUESTS: Final = ( + pytest.param({}, id="summary_route"), + pytest.param({"context_management": [{"type": "compaction", "compact_threshold": 50000}]}, id="native_route"), +) + + +async def _stored_turns_the_compaction_replaces(previous_response_id: str) -> ChatCompletionSession: + return ChatCompletionSession( + messages=[ + {"role": "user", "content": "Weather in SF?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "get_weather", "arguments": "{}"}} + ], + }, + ], + litellm_session_id=None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("compaction_request", _COMPACTION_REQUESTS) +async def test_claude_count_request_counts_a_compaction_item_as_its_summary_text( + compaction_request: dict[str, JsonValue], +) -> None: + request: Final = await chat_count_request( + model="claude-opus-5-5", + custom_llm_provider="anthropic", + api_base=None, + responses_request={"model": "claude-opus-5-5", "input": _COMPACTED_INPUT, **compaction_request}, + policy=DropPolicy(drop_params=False, additional_drop_params=frozenset()), + ) + + assert isinstance(request, ChatCountRequest) + assert [(message.get("role"), message.get("content")) for message in request.messages] == [ + ("user", _SUMMARY), + ("user", "And Osaka?"), + ] + assert "compaction_blocks" not in json.dumps(request.messages, default=str) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("compaction_request", _COMPACTION_REQUESTS) +async def test_claude_count_request_leaves_out_the_stored_turns_a_compaction_item_replaces( + compaction_request: dict[str, JsonValue], +) -> None: + request: Final = await chat_count_request( + model="claude-opus-5-5", + custom_llm_provider="anthropic", + api_base=None, + responses_request={ + "model": "claude-opus-5-5", + "previous_response_id": "resp_before_the_compaction", + "input": _COMPACTED_INPUT, + **compaction_request, + }, + policy=DropPolicy(drop_params=False, additional_drop_params=frozenset()), + session_lookup=_stored_turns_the_compaction_replaces, + ) + + assert isinstance(request, ChatCountRequest) + counted: Final = json.dumps(request.messages, default=str) + assert _SUMMARY in counted and "And Osaka?" in counted + assert "Weather in SF?" not in counted + assert "call_0" not in counted diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index f34ca7d0ff8..30e13442c92 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, anthropic_messages_pt, ) +from litellm.responses.litellm_completion_transformation.reasoning_items import encode_thinking_blocks from litellm.responses.litellm_completion_transformation.transformation import ( TOOL_CALLS_CACHE, ChatCompletionSession, @@ -6502,6 +6503,363 @@ def test_transform_request_reads_service_tier_from_responses_api_request() -> No assert result["service_tier"] == "priority" +_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "Earlier turns covered the Tokyo trip."} + + +_COMPACTION_REQUESTED: Final = ResponsesAPIOptionalRequestParams( + context_management=[{"type": "compaction", "compact_threshold": 50000}] +) +_SUMMARY_AS_TEXT: Final = {"role": "user", "content": _COMPACTION_BLOCK["content"]} + + +def _anthropic_form_context_management(edit: Mapping[str, object]) -> ResponsesAPIOptionalRequestParams: + return cast( # cast-ok: Anthropic's native {"edits": [...]} form, which the param type does not declare + ResponsesAPIOptionalRequestParams, {"context_management": {"edits": [edit]}} + ) + + +@pytest.mark.parametrize( + ("model", "responses_api_request", "replayed_compaction"), + [ + ( + "anthropic/claude-sonnet-5-5", + _COMPACTION_REQUESTED, + { + "role": "assistant", + "content": None, + "provider_specific_fields": {"compaction_blocks": [_COMPACTION_BLOCK]}, + }, + ), + ( + "anthropic/claude-sonnet-5-5", + _anthropic_form_context_management({"type": "compact_20260112"}), + { + "role": "assistant", + "content": None, + "provider_specific_fields": {"compaction_blocks": [_COMPACTION_BLOCK]}, + }, + ), + ("anthropic/claude-sonnet-5-5", ResponsesAPIOptionalRequestParams(), _SUMMARY_AS_TEXT), + ( + "anthropic/claude-sonnet-5-5", + _anthropic_form_context_management({"type": "clear_tool_uses_20250919"}), + _SUMMARY_AS_TEXT, + ), + ("gemini/gemini-3.8-flash", _COMPACTION_REQUESTED, _SUMMARY_AS_TEXT), + ("bedrock/claude_platform/claude-sonnet-5-5", _COMPACTION_REQUESTED, _SUMMARY_AS_TEXT), + ], +) +def test_compaction_input_item_replays_as_a_compaction_block_only_when_anthropic_compacts( + model: str, responses_api_request: ResponsesAPIOptionalRequestParams, replayed_compaction: Mapping[str, object] +) -> None: + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model=model, + input=cast( + ResponseInputParam, + [ + {"type": "compaction", "id": "cmp_openai", "encrypted_content": "gAAAAABopaque"}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"role": "user", "content": "And Osaka?"}, + ], + ), + responses_api_request=responses_api_request, + ) + + assert [dict(message) for message in result["messages"]] == [ + replayed_compaction, + {"role": "user", "content": "And Osaka?"}, + ] + + +def _replayed_compacted_answer(model: str, context_management: object) -> list[tuple[object, object, object]]: + async def session_lookup(previous_response_id: str) -> ChatCompletionSession: + return ChatCompletionSession( + messages=[ + Message( + role="assistant", + content="Day one, Asakusa.", + provider_specific_fields={"compaction_blocks": [_COMPACTION_BLOCK]}, + ), + {"role": "user", "content": "Day two?"}, + Message(role="assistant", content="Ueno."), + ], + litellm_session_id=None, + ) + + request: Final = asyncio.run( + LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( + previous_response_id="resp_compacted", + litellm_completion_request={ + "model": model, + "messages": [{"role": "user", "content": "And Osaka?"}], + **({"context_management": context_management} if context_management else {}), + }, + instructions=None, + session_lookup=session_lookup, + ) + ) + return [ + ( + message.get("role"), + message.get("content"), + (message.get("provider_specific_fields") or {}).get("compaction_blocks"), + ) + for message in request["messages"] + ] + + +_TURNS_AFTER_THE_COMPACTED_ANSWER: Final = [ + ("user", "Day two?", None), + ("assistant", "Ueno.", None), + ("user", "And Osaka?", None), +] +_ANSWER_AFTER_SUMMARY: Final = [ + ("user", _COMPACTION_BLOCK["content"], None), + ("assistant", "Day one, Asakusa.", None), + *_TURNS_AFTER_THE_COMPACTED_ANSWER, +] + + +@pytest.mark.parametrize( + ("model", "context_management", "replayed"), + [ + ( + "anthropic/claude-sonnet-5-5", + _COMPACTION_REQUESTED["context_management"], + [("assistant", "Day one, Asakusa.", [_COMPACTION_BLOCK]), *_TURNS_AFTER_THE_COMPACTED_ANSWER], + ), + ("anthropic/claude-sonnet-5-5", None, _ANSWER_AFTER_SUMMARY), + ( + "bedrock/claude_platform/claude-sonnet-5-5", + _COMPACTION_REQUESTED["context_management"], + _ANSWER_AFTER_SUMMARY, + ), + ("gpt-5.6", _COMPACTION_REQUESTED["context_management"], _ANSWER_AFTER_SUMMARY), + ], +) +def test_stored_compacted_answer_replays_its_block_only_when_anthropic_compacts( + model: str, context_management: object, replayed: list[tuple[object, object, object]] +) -> None: + assert _replayed_compacted_answer(model, context_management) == replayed + + +def test_summary_replay_drops_the_history_its_compaction_item_replaces() -> None: + result: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini/gemini-3.8-flash", + input=cast( + ResponseInputParam, + [ + {"role": "system", "content": "Answer briefly."}, + {"role": "user", "content": "Plan a Tokyo trip."}, + {"role": "assistant", "content": "Day one, Asakusa."}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"role": "user", "content": "And Osaka?"}, + ], + ), + responses_api_request=_COMPACTION_REQUESTED, + ) + + assert [dict(message) for message in result["messages"]] == [ + {"role": "system", "content": "Answer briefly."}, + _SUMMARY_AS_TEXT, + {"role": "user", "content": "And Osaka?"}, + ] + + + +@pytest.mark.parametrize( + ("call", "output"), + [ + pytest.param( + {"type": "function_call", "call_id": "call_weather", "name": "get_weather", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_weather", "output": "Rain."}, + id="function_call", + ), + pytest.param( + {"type": "custom_tool_call", "call_id": "call_patch", "name": "apply_patch", "input": "*** Begin Patch"}, + {"type": "custom_tool_call_output", "call_id": "call_patch", "output": "Done."}, + id="custom_tool_call", + ), + ], +) +def test_summary_replay_keeps_a_call_its_compaction_item_replaces_when_a_later_output_answers_it( + call: Mapping[str, str], output: Mapping[str, str] +) -> None: + messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="gemini/gemini-3.8-flash", + input=cast( + ResponseInputParam, + [ + {"role": "system", "content": "Answer briefly."}, + {"role": "user", "content": "Plan a Tokyo trip."}, + call, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + output, + ], + ), + responses_api_request=_COMPACTION_REQUESTED, + )["messages"] + + assert [message["role"] for message in messages] == ["system", "user", "assistant", "tool"] + assert dict(messages[1]) == _SUMMARY_AS_TEXT + assert [tool_call["id"] for tool_call in messages[2]["tool_calls"]] == [output["call_id"]] + assert messages[3]["tool_call_id"] == output["call_id"] + + +_ANSWERED_THINKING: Final = {"type": "thinking", "thinking": "Need Osaka weather.", "signature": "sig_osaka"} +_CHECKING_OSAKA: Final = { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "Checking Osaka."}], +} + + +def _reasoning_item(item_id: str, thinking: Mapping[str, str]) -> Mapping[str, object]: + return {"type": "reasoning", "id": item_id, "summary": [], "encrypted_content": encode_thinking_blocks([thinking])} + + +def test_summary_replay_keeps_only_the_reasoning_of_the_turn_whose_call_a_later_output_answers() -> None: + tokyo_thinking: Final = {"type": "thinking", "thinking": "Need Tokyo weather.", "signature": "sig_tokyo"} + messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input=cast( + ResponseInputParam, + [ + {"role": "user", "content": "Plan a Tokyo trip."}, + _reasoning_item("rs_tokyo", tokyo_thinking), + {"type": "function_call", "call_id": "call_tokyo", "name": "get_weather", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_tokyo", "output": "Sun."}, + _reasoning_item("rs_osaka", _ANSWERED_THINKING), + _CHECKING_OSAKA, + {"type": "function_call", "call_id": "call_osaka", "name": "get_weather", "arguments": "{}"}, + {"type": "function_call", "call_id": "call_kyoto", "name": "get_weather", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_kyoto", "output": "Clouds."}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"type": "function_call_output", "call_id": "call_osaka", "output": "Rain."}, + ], + ), + responses_api_request=ResponsesAPIOptionalRequestParams(), + )["messages"] + + assert [message["role"] for message in messages] == ["user", "assistant", "tool"] + assert dict(messages[0]) == _SUMMARY_AS_TEXT + assert messages[1]["thinking_blocks"] == [_ANSWERED_THINKING] + assert [tool_call["id"] for tool_call in messages[1]["tool_calls"]] == ["call_osaka"] + + +def test_native_compaction_replay_moves_a_call_its_compaction_item_splits_from_its_output_after_the_block() -> None: + messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input=cast( + ResponseInputParam, + [ + {"role": "user", "content": "Plan a Tokyo trip."}, + _reasoning_item("rs_osaka", _ANSWERED_THINKING), + _CHECKING_OSAKA, + {"type": "function_call", "call_id": "call_osaka", "name": "get_weather", "arguments": "{}"}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"type": "function_call_output", "call_id": "call_osaka", "output": "Rain."}, + ], + ), + responses_api_request=_COMPACTION_REQUESTED, + )["messages"] + + assert [message["role"] for message in messages] == ["user", "assistant", "assistant", "tool"] + assert messages[0]["content"] == "Plan a Tokyo trip." + assert messages[1]["provider_specific_fields"] == {"compaction_blocks": [_COMPACTION_BLOCK]} + assert messages[2]["thinking_blocks"] == [_ANSWERED_THINKING] + assert [tool_call["id"] for tool_call in messages[2]["tool_calls"]] == ["call_osaka"] + assert messages[3]["tool_call_id"] == "call_osaka" + + +@pytest.mark.parametrize( + "responses_api_request", + [ + pytest.param(_COMPACTION_REQUESTED, id="native_compaction_block"), + pytest.param(ResponsesAPIOptionalRequestParams(), id="summary_as_text"), + ], +) +def test_compaction_replay_invents_no_call_for_an_output_whose_call_was_never_stored( + responses_api_request: ResponsesAPIOptionalRequestParams, +) -> None: + async def no_stored_turns(previous_response_id: str) -> ChatCompletionSession: + return ChatCompletionSession(messages=[], litellm_session_id=None) + + chat_request: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input=cast( + ResponseInputParam, + [ + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"type": "function_call_output", "call_id": "call_never_stored", "output": "Rain."}, + ], + ), + responses_api_request=ResponsesAPIOptionalRequestParams( + **responses_api_request, + tools=[{"type": "function", "name": "get_weather", "parameters": {"type": "object"}, "strict": False}], + ), + ) + + request: Final = asyncio.run( + LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( + previous_response_id="resp_unknown", + litellm_completion_request=chat_request, + instructions=None, + session_lookup=no_stored_turns, + ) + ) + + assert not any(message.get("tool_calls") for message in request["messages"]) + + +@pytest.mark.parametrize( + ("responses_api_request", "expected_turns"), + [ + pytest.param( + _COMPACTION_REQUESTED, + [("user", ()), ("assistant", ("call_0",)), ("tool", "Sunny."), ("assistant", ()), ("assistant", ("call_0",)), ("tool", "Rain.")], + id="native_compaction_block", + ), + pytest.param( + ResponsesAPIOptionalRequestParams(), + [("user", ()), ("assistant", ("call_0",)), ("tool", "Rain.")], + id="summary_as_text", + ), + ], +) +def test_compaction_replay_moves_only_the_call_a_later_output_answers_when_an_earlier_call_reused_its_id( + responses_api_request: ResponsesAPIOptionalRequestParams, expected_turns: list[tuple[str, object]] +) -> None: + messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_request_to_chat_completion_request( + model="anthropic/claude-sonnet-5-5", + input=cast( + ResponseInputParam, + [ + {"role": "user", "content": "Plan a Tokyo trip."}, + {"type": "function_call", "call_id": "call_0", "name": "get_weather", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_0", "output": "Sunny."}, + _reasoning_item("rs_osaka", _ANSWERED_THINKING), + _CHECKING_OSAKA, + {"type": "function_call", "call_id": "call_0", "name": "get_weather", "arguments": "{}"}, + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"type": "function_call_output", "call_id": "call_0", "output": "Rain."}, + ], + ), + responses_api_request=responses_api_request, + )["messages"] + + assert [ + ( + message["role"], + message["content"] + if message["role"] == "tool" + else tuple(tool_call["id"] for tool_call in message.get("tool_calls") or ()), + ) + for message in messages + ] == expected_turns + assert messages[-2]["thinking_blocks"] == [_ANSWERED_THINKING] + + def _continued_chain_request(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: async def session_lookup(previous_response_id: str) -> ChatCompletionSession: assert previous_response_id == "resp_turn_1" @@ -6603,3 +6961,47 @@ async def test_bridge_partial_response_on_disconnect_bills_the_relayed_chat_chun assert incomplete_event.response.usage is not None assert incomplete_event.response.usage.output_tokens == litellm.token_counter(model="gpt-4o-mini", text=relayed_text) assert incomplete_event.response.usage.input_tokens > 0 + + +async def _input_tokens_billed_on_disconnect(request_input: ResponseInputParam) -> int: + from litellm.responses.litellm_completion_transformation.streaming_iterator import ( + LiteLLMCompletionStreamingIterator, + ) + + chat_stream: Final = await litellm.acompletion( + model="anthropic/claude-sonnet-5-5", + messages=[{"role": "user", "content": "Which day is driest?"}], + mock_response="Thursday looks driest, so plan the walking tour then.", + stream=True, + api_key="test-key", + ) + iterator: Final = LiteLLMCompletionStreamingIterator( + model="anthropic/claude-sonnet-5-5", + litellm_custom_stream_wrapper=chat_stream, + request_input=request_input, + responses_api_request={}, + custom_llm_provider="anthropic", + ) + _ = [await iterator.__anext__() for _ in range(6)] + incomplete_event: Final = iterator.partial_response_on_disconnect() + assert incomplete_event is not None and incomplete_event.response.usage is not None + return incomplete_event.response.usage.input_tokens + + +@pytest.mark.asyncio +async def test_bridge_partial_response_on_disconnect_bills_only_the_input_since_the_newest_compaction_item(): + since_compaction: Final = cast( + ResponseInputParam, + [ + {"type": "compaction", "id": "cmp_litellm", "encrypted_content": json.dumps(_COMPACTION_BLOCK)}, + {"role": "user", "content": "Which day is driest?"}, + ], + ) + history_it_replaces: Final = cast( + ResponseInputParam, + [{"role": "user", "content": "Plan a Tokyo trip, day by day, with every museum and food stall. " * 200}], + ) + + assert await _input_tokens_billed_on_disconnect( + cast(ResponseInputParam, [*history_it_replaces, *since_compaction]) + ) == await _input_tokens_billed_on_disconnect(since_compaction) diff --git a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py index 7a146c1b665..0eb1aa26a65 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_session_handler.py +++ b/tests/unit/responses/litellm_completion_transformation/test_session_handler.py @@ -1,6 +1,7 @@ import json import sqlite3 from collections.abc import Sequence +from itertools import chain from types import SimpleNamespace from typing import Final from unittest.mock import AsyncMock, patch @@ -18,6 +19,7 @@ from litellm.responses.litellm_completion_transformation.session_handler import ResponsesSessionHandler, _normalize_redacted_tool_call_arguments, ) +from litellm.responses.litellm_completion_transformation.transformation import ChatCompletionSession from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.responses.utils import ResponsesAPIRequestUtils from litellm.types.llms.openai import ResponsesAPIResponse @@ -233,11 +235,10 @@ async def test_e2e_cold_storage_successful_retrieval(): patch.object(session_handler, "COLD_STORAGE_HANDLER") as mock_cold_storage, patch("litellm.cold_storage_custom_logger", return_value="s3"), ): - # Setup mocks mock_get_spend_logs.return_value = mock_spend_logs - mock_cold_storage.get_proxy_server_request_from_cold_storage_with_object_key = ( - AsyncMock(return_value=full_proxy_request) + mock_cold_storage.get_proxy_server_request_from_cold_storage_with_object_key = AsyncMock( + return_value=full_proxy_request ) # Call the main function @@ -291,7 +292,6 @@ async def test_e2e_cold_storage_fallback_to_truncated_payload(): ) as mock_get_spend_logs, patch.object(session_handler, "COLD_STORAGE_HANDLER") as mock_cold_storage, ): - # Setup mocks mock_get_spend_logs.return_value = mock_spend_logs @@ -358,41 +358,25 @@ async def test_should_check_cold_storage_for_full_payload(): with patch("litellm.cold_storage_custom_logger", return_value="s3"): # Test case 1: Should return True for truncated content - result1 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload( - proxy_request_with_truncated_pdf - ) - assert ( - result1 == True - ), "Should return True for proxy request with truncated PDF content" + result1 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_request_with_truncated_pdf) + assert result1 == True, "Should return True for proxy request with truncated PDF content" # Test case 2: Should return False for regular content - result2 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload( - proxy_request_regular - ) - assert ( - result2 == False - ), "Should return False for regular proxy request without truncation" + result2 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_request_regular) + assert result2 == False, "Should return False for regular proxy request without truncation" # Test case 3: Should return True for empty request - result3 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload( - proxy_request_empty - ) + result3 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_request_empty) assert result3 == True, "Should return True for empty proxy request" # Test case 4: Should return True for None request - result4 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload( - proxy_request_none - ) + result4 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_request_none) assert result4 == True, "Should return True for None proxy request" # Test case 5: Should return False when cold storage is not configured with patch.object(litellm, "cold_storage_custom_logger", None): - result5 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload( - proxy_request_with_truncated_pdf - ) - assert ( - result5 == False - ), "Should return False when cold storage is not configured, even with truncated content" + result5 = ResponsesSessionHandler._should_check_cold_storage_for_full_payload(proxy_request_with_truncated_pdf) + assert result5 == False, "Should return False when cold storage is not configured, even with truncated content" @pytest.mark.asyncio @@ -422,9 +406,7 @@ async def test_get_chat_completion_message_history_empty_response_dict(): } ] - with patch.object( - ResponsesSessionHandler, "get_all_spend_logs_for_previous_response_id" - ) as mock_get_spend_logs: + with patch.object(ResponsesSessionHandler, "get_all_spend_logs_for_previous_response_id") as mock_get_spend_logs: mock_get_spend_logs.return_value = mock_spend_logs # Call the function @@ -518,9 +500,7 @@ async def test_message_history_reconstructs_list_shaped_input(): ) as mock_get_spend_logs: mock_get_spend_logs.return_value = mock_spend_logs - result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( - request_id - ) + result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(request_id) messages = result["messages"] assert [(message.get("role"), message.get("content")) for message in messages] == [ @@ -549,9 +529,7 @@ async def test_message_history_retries_a_spend_log_the_batch_writer_has_not_flus fake_prisma_client = _FakePrismaClient(results=[[], [spend_log]]) with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client): - result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( - request_id - ) + result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id(request_id) messages = result["messages"] assert [(message.get("role"), message.get("content")) for message in messages] == [ @@ -618,14 +596,10 @@ async def test_message_history_looks_up_the_decoded_chat_completion_id(): model_id="e0f302a1412e78470ebb28cbed01fff5f88c0d331c667e9f2ba4b413c6fbd282", response_id=request_id, ) - fake_prisma_client = _FakePrismaClient( - results=[[_spend_log(request_id, "session-a", "Hello.", "Hi.")]] - ) + fake_prisma_client = _FakePrismaClient(results=[[_spend_log(request_id, "session-a", "Hello.", "Hi.")]]) with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client): - await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( - encoded_response_id - ) + await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id(encoded_response_id) assert len(fake_prisma_client.db.calls) == 1 assert request_id in fake_prisma_client.db.calls[0] @@ -724,8 +698,9 @@ async def test_session_lookup_does_not_retry_when_spend_logs_are_disabled( """ fake_prisma_client = _FakePrismaClient(results=[]) - with patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client), patch( - "litellm.proxy.proxy_server.disable_spend_logs", True + with ( + patch("litellm.proxy.proxy_server.prisma_client", fake_prisma_client), + patch("litellm.proxy.proxy_server.disable_spend_logs", True), ): spend_logs = await ResponsesSessionHandler.get_all_spend_logs_for_previous_response_id( "chatcmpl-does-not-exist" @@ -741,7 +716,11 @@ def test_normalize_redacted_arguments_skips_custom_tool_calls(): content=None, tool_calls=[ {"id": "call_c", "type": "custom", "custom": {"name": "run_code", "input": "print(1)"}}, - {"id": "call_f", "type": "function", "function": {"name": "get_weather", "arguments": "redacted-by-litellm"}}, + { + "id": "call_f", + "type": "function", + "function": {"name": "get_weather", "arguments": "redacted-by-litellm"}, + }, ], ) @@ -796,11 +775,13 @@ async def test_message_history_normalizes_redacted_tool_call_arguments(): } ] - with patch.object( # test-quality-ok: the handler has no DI seam for the spend-log fetch; every test in this file stubs this same boundary - ResponsesSessionHandler, - "get_all_spend_logs_for_previous_response_id", - new_callable=AsyncMock, - ) as mock_get_spend_logs: + with ( + patch.object( # test-quality-ok: the handler has no DI seam for the spend-log fetch; every test in this file stubs this same boundary + ResponsesSessionHandler, + "get_all_spend_logs_for_previous_response_id", + new_callable=AsyncMock, + ) as mock_get_spend_logs + ): mock_get_spend_logs.return_value = mock_spend_logs result = await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( @@ -878,6 +859,286 @@ async def test_message_history_replays_real_key_named_tool_payloads() -> None: assert json.loads(tool_message["content"]) == function_output +def _turn_spend_log( + turn_input: str | list[dict[str, str]], answer: str, compaction_blocks: list[dict[str, str]] | None = None +) -> dict[str, object]: + compaction_fields: Final = ( + {"provider_specific_fields": {"compaction_blocks": compaction_blocks}} if compaction_blocks else {} + ) + return { + "request_id": f"resp_{len(turn_input)}", + "session_id": "session-compacted", + "proxy_server_request": json.dumps({"input": turn_input}), + "response": { + "id": f"chatcmpl-{len(turn_input)}", + "object": "chat.completion", + "choices": [{"index": 0, "message": {"role": "assistant", "content": answer, **compaction_fields}}], + }, + } + + +def _role_and_content(message: object) -> tuple[object, object]: + if isinstance(message, dict): + return message.get("role"), message.get("content") + return getattr(message, "role", None), getattr(message, "content", None) + + +async def _history_from_stored_turns( + spend_logs: tuple[dict[str, object], ...], previous_response_id: str +) -> ChatCompletionSession: + with patch("litellm.proxy.proxy_server.prisma_client", SimpleNamespace(db=_StoredSpendLogs(spend_logs))): + return await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( + previous_response_id + ) + + +@pytest.mark.asyncio +async def test_previous_response_history_starts_at_the_newest_turn_that_sent_a_compaction_item(): + first_turn: Final = [{"role": "user", "content": "Plan a Tokyo trip."}] + compaction_item: Final = { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": "Turn one planned Tokyo."}), + } + second_turn: Final = [ + *first_turn, + {"role": "assistant", "content": "Day one, Asakusa."}, + compaction_item, + {"role": "user", "content": "And Osaka?"}, + ] + spend_logs: Final = ( + _turn_spend_log(first_turn, "Day one, Asakusa."), + _turn_spend_log(second_turn, "Dotonbori."), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_4") + + assert [_role_and_content(message) for message in session["messages"]] == [ + ("user", "Turn one planned Tokyo."), + ("user", "And Osaka?"), + ("assistant", "Dotonbori."), + ] + + +@pytest.mark.parametrize( + "first_turn_input", + [pytest.param([{"role": "user", "content": "Hi."}], id="item_list"), pytest.param("Hi.", id="plain_string")], +) +@pytest.mark.asyncio +async def test_previous_response_history_starts_at_the_newest_answer_that_compacted( + first_turn_input: str | list[dict[str, str]], +): + compaction_blocks: Final = [{"type": "compaction", "content": "Turn one planned Tokyo."}] + spend_logs: Final = ( + _turn_spend_log(first_turn_input, "Hello."), + _turn_spend_log( + [{"role": "developer", "content": "Answer briefly."}, {"role": "user", "content": "Plan a Tokyo trip."}], + "Day one, Asakusa.", + compaction_blocks, + ), + _turn_spend_log([{"role": "user", "content": "And Osaka?"}], "Dotonbori."), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_1") + + assert [_role_and_content(message) for message in session["messages"]] == [ + ("developer", "Answer briefly."), + ("assistant", "Day one, Asakusa."), + ("user", "And Osaka?"), + ("assistant", "Dotonbori."), + ] + assert getattr(session["messages"][1], "provider_specific_fields", None) == {"compaction_blocks": compaction_blocks} + + +_COMPACTION_ITEM_SENT: Final = { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": "Turn one planned Tokyo."}), +} + + +@pytest.mark.parametrize( + "compacting_turn", + [ + pytest.param( + _turn_spend_log( + [{"role": "user", "content": "Plan a Tokyo trip."}], + "Day one, Asakusa.", + [{"type": "compaction", "content": "Turn one planned Tokyo."}], + ), + id="answer_compacted", + ), + pytest.param( + _turn_spend_log( + [_COMPACTION_ITEM_SENT, {"role": "user", "content": "Plan a Tokyo trip."}], "Day one, Asakusa." + ), + id="input_sent_a_compaction_item", + ), + ], +) +@pytest.mark.asyncio +async def test_previous_response_history_keeps_instructions_from_turns_the_compaction_covers( + compacting_turn: dict[str, object], +): + spend_logs: Final = ( + _turn_spend_log( + [{"role": "developer", "content": "Use metric units."}, {"role": "user", "content": "Hi."}], "Hello." + ), + compacting_turn, + _turn_spend_log([{"role": "user", "content": "And Osaka?"}], "Dotonbori."), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_1") + + roles_and_contents: Final = [_role_and_content(message) for message in session["messages"]] + assert roles_and_contents[0] == ("developer", "Use metric units.") + assert ("user", "Hi.") not in roles_and_contents + assert ("assistant", "Hello.") not in roles_and_contents + assert roles_and_contents[-2:] == [("user", "And Osaka?"), ("assistant", "Dotonbori.")] + + +@pytest.mark.asyncio +async def test_previous_response_history_drops_a_tool_output_carrying_a_system_role_the_compaction_covers(): + spend_logs: Final = ( + _turn_spend_log( + [ + {"role": "user", "content": "Look up the code."}, + {"type": "function_call", "call_id": "call_1", "name": "lookup", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "4242", "role": "system"}, + ], + "Found it.", + ), + _turn_spend_log([_COMPACTION_ITEM_SENT, {"role": "user", "content": "Plan a Tokyo trip."}], "Day one."), + _turn_spend_log([{"role": "user", "content": "And Osaka?"}], "Dotonbori."), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_1") + + contents: Final = [str(_role_and_content(message)[1]) for message in session["messages"]] + assert not any("4242" in content for content in contents) + assert contents[-2:] == ["And Osaka?", "Dotonbori."] + + +def _output_message(text: str) -> dict[str, object]: + return {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]} + + +@pytest.mark.asyncio +async def test_previous_response_history_starts_at_a_compaction_item_in_a_stored_responses_output(): + responses_shaped_turn: Final = { + "request_id": "resp_mcp", + "session_id": "session-compacted", + "proxy_server_request": json.dumps({"input": [{"role": "user", "content": "Plan a Tokyo trip."}]}), + "response": { + "id": "resp_mcp", + "object": "response", + "output": [_output_message("Checking the map."), _COMPACTION_ITEM_SENT, _output_message("Day one.")], + }, + } + spend_logs: Final = ( + _turn_spend_log([{"role": "user", "content": "Hi."}], "Hello."), + responses_shaped_turn, + _turn_spend_log([{"role": "user", "content": "And Osaka?"}], "Dotonbori."), + ) + + session: Final = await _history_from_stored_turns(spend_logs, "resp_1") + + replayed: Final = json.dumps([_role_and_content(message) for message in session["messages"]], default=str) + assert "Hello." not in replayed + assert "Plan a Tokyo trip." not in replayed + assert "Checking the map." not in replayed + assert "Turn one planned Tokyo." in replayed + assert [str(_role_and_content(message)[1]) for message in session["messages"]][-2:] == [ + "And Osaka?", + "Dotonbori.", + ] + + +def _tool_call_ids(message: object) -> tuple[str, ...]: + tool_calls: Final = message.get("tool_calls") if isinstance(message, dict) else getattr(message, "tool_calls", None) + return tuple(call["id"] if isinstance(call, dict) else call.id for call in tool_calls or ()) + + +_WEATHER_CALL_TURN: Final = { + "request_id": "resp_weather", + "session_id": "session-compacted", + "proxy_server_request": json.dumps({"input": [{"role": "user", "content": "Weather in Paris?"}]}), + "response": { + "id": "chatcmpl-weather", + "object": "chat.completion", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}, + } + ], + }, + } + ], + }, +} + + +def _compacted_weather_answer(request_id: str) -> dict[str, object]: + return { + **_turn_spend_log( + [_COMPACTION_ITEM_SENT, {"type": "function_call_output", "call_id": "call_1", "output": "sunny"}], + "It is sunny.", + ), + "request_id": request_id, + } + + +@pytest.mark.asyncio +async def test_previous_response_history_ignores_a_compaction_item_sent_after_the_referenced_response(): + session: Final = await _history_from_stored_turns( + (_WEATHER_CALL_TURN, _compacted_weather_answer("resp_first_send")), "resp_weather" + ) + + assert _role_and_content(session["messages"][0]) == ("user", "Weather in Paris?") + assert tuple(chain.from_iterable(map(_tool_call_ids, session["messages"]))) == ("call_1",) + + +@pytest.mark.asyncio +async def test_a_turn_after_a_compaction_item_keeps_the_stored_call_its_tool_output_answers(): + session: Final = await _history_from_stored_turns( + (_WEATHER_CALL_TURN, _compacted_weather_answer("resp_compacted")), "resp_compacted" + ) + + roles: Final = [_role_and_content(message)[0] for message in session["messages"]] + assert roles == ["user", "assistant", "tool", "assistant"] + assert _tool_call_ids(session["messages"][1]) == ("call_1",) + assert ("user", "Weather in Paris?") not in map(_role_and_content, session["messages"]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("spend_logs", "previous_response_id", "replayed_roles"), + [ + ((_WEATHER_CALL_TURN, _compacted_weather_answer("resp_first_send")), "resp_weather", ["user", "assistant"]), + ( + (_WEATHER_CALL_TURN, _compacted_weather_answer("resp_first_send"), _compacted_weather_answer("resp_retry")), + "resp_first_send", + ["user", "assistant", "tool", "assistant"], + ), + ], +) +async def test_a_compacted_session_replays_no_turn_sent_after_the_referenced_response( + spend_logs: tuple[dict[str, object], ...], previous_response_id: str, replayed_roles: list[str] +): + session: Final = await _history_from_stored_turns(spend_logs, previous_response_id) + + assert [_role_and_content(message)[0] for message in session["messages"]] == replayed_roles + + class _SqliteBackedPrismaDB: def __init__(self, rows: list[tuple[str, str, str, str, str | None, str | None]]) -> None: self._connection = sqlite3.connect(":memory:") @@ -889,7 +1150,9 @@ class _SqliteBackedPrismaDB: self._connection.executemany('INSERT INTO "LiteLLM_SpendLogs" VALUES (?, ?, ?, ?, ?, ?)', rows) async def query_raw(self, query: str, *args: object) -> list[dict[str, object]]: - sqlite_query: Final = query.replace("= ANY($1::text[])", "IN (SELECT value FROM json_each(?))").replace("$1", "?") + sqlite_query: Final = query.replace("= ANY($1::text[])", "IN (SELECT value FROM json_each(?))").replace( + "$1", "?" + ) sqlite_args: Final = tuple(json.dumps(arg) if isinstance(arg, list) else arg for arg in args) return [dict(row) for row in self._connection.execute(sqlite_query, sqlite_args)] diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index 1a3430224d1..ed90acbd2fa 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -1524,3 +1524,37 @@ async def test_caller_function_named_like_the_search_tool_streams_as_function_ca assert [(item["type"], item["name"]) for item in output] == [("function_call", "tool_search_tool_regex")] assert "response.function_call_arguments.done" in [event["type"] for event in events] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_a_compaction_block_without_its_summary_streams_no_compaction_item(sync_mode: bool): + deltas: Final = ( + Delta( + role="assistant", + content="", + provider_specific_fields={"compaction_blocks": [{"type": "compaction", "content": None}]}, + ), + Delta(content="Ok."), + ) + chunks: Final = tuple( + ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-sonnet-5-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices(index=0, delta=delta, finish_reason="stop" if index == len(deltas) - 1 else None) + ], + ) + for index, delta in enumerate(deltas) + ) + iterator: Final = _build_iterator(chunks) + events: Final = ( + [event.model_dump(exclude_none=True) for event in iterator] + if sync_mode + else [event.model_dump(exclude_none=True) async for event in iterator] + ) + + assert [event["item"]["type"] for event in events if "item" in event] == ["message", "message"] + assert [item["type"] for item in events[-1]["response"]["output"]] == ["message"] diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index ea3eab7e002..436367bd82d 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -7,6 +7,7 @@ in expected_responses_api_request/. import copy import json import logging +from collections.abc import Awaitable, Callable from pathlib import Path from importlib import import_module from typing import Final @@ -1141,6 +1142,61 @@ async def test_a_native_responses_provider_places_every_point_itself(): assert with_unmatchable_system == user_only +_CLAUDE_COMPACTION_ITEM: Final = { + "type": "compaction", + "id": "cmp_claude", + "encrypted_content": json.dumps({"type": "compaction", "content": "The user's cat is named Biscuit."}), +} +_OPENAI_COMPACTION_ITEM: Final = {"type": "compaction", "id": "cmp_openai", "encrypted_content": "gAAAAABopaque"} +_HISTORY: Final = ( + {"role": "system", "content": "Answer briefly."}, + {"role": "user", "content": "My cat is named Biscuit."}, + {"role": "assistant", "content": "Noted."}, +) +_NEXT_TURN: Final = {"role": "user", "content": "What is my cat's name?"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("call", "upstream_url"), + [ + pytest.param(litellm.aresponses, "https://api.openai.com/v1/responses", id="create"), + pytest.param(litellm.acompact_responses, "https://api.openai.com/v1/responses/compact", id="compact"), + ], +) +@pytest.mark.parametrize( + ("compaction_item", "sent_input"), + [ + ( + _CLAUDE_COMPACTION_ITEM, + [ + _HISTORY[0], + {"role": "user", "content": "The user's cat is named Biscuit.", "type": "message"}, + _NEXT_TURN, + ], + ), + (_OPENAI_COMPACTION_ITEM, [*_HISTORY, _OPENAI_COMPACTION_ITEM, _NEXT_TURN]), + ], +) +async def test_aresponses_sends_a_claude_compaction_item_to_openai_as_its_summary( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + call: Callable[..., Awaitable[object]], + upstream_url: str, + compaction_item: dict[str, str], + sent_input: list[dict[str, str]], +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post(upstream_url).mock( + return_value=httpx.Response(200, json=_minimal_responses_api_payload("resp_compacted", "gpt-5.6")) + ) + + await call(model="openai/gpt-5.6", api_key="fake-api-key", input=[*_HISTORY, compaction_item, _NEXT_TURN]) + + assert json.loads(upstream.calls[0].request.read())["input"] == sent_input + + def test_non_streamed_response_ignores_a_stream_timeout_it_never_uses() -> None: answer: Final = json.dumps(_minimal_responses_api_payload("resp_unstreamed", "gpt-5.6")).encode() client: Final = HTTPHandler( diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 62d89a55e85..6c3e6177503 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -2605,6 +2605,27 @@ def test_faked_stream_builds_each_refusal_summary_and_custom_tool_input_once_for } +_COMPACTION_SUMMARY: Final = "The user planned a Tokyo trip." +_NEXT_TURN: Final = {"type": "message", "role": "user", "content": "And Osaka?"} +_COMPACTED_INPUT: Final = [ + {"type": "message", "role": "user", "content": "Plan a Tokyo trip. " * 200}, + {"type": "message", "role": "assistant", "content": "Day one, Asakusa. " * 200}, + { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": _COMPACTION_SUMMARY}), + }, + _NEXT_TURN, +] +_SUMMARY_INPUT: Final = [{"type": "message", "role": "user", "content": _COMPACTION_SUMMARY}, _NEXT_TURN] + + +def test_usage_estimate_counts_a_compaction_item_as_its_summary(): + assert _estimate_usage_from_text("gpt-4o-mini", _COMPACTED_INPUT, {}, "Dotonbori.") == _estimate_usage_from_text( + "gpt-4o-mini", _SUMMARY_INPUT, {}, "Dotonbori." + ) + + @pytest.mark.asyncio async def test_shutdown_drain_waits_for_a_pending_logging_task(): release: Final = asyncio.Event() diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index e1f748fcb58..9cc1bf6ce79 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -7889,6 +7889,29 @@ def test_count_pre_call_check_tokens_across_api_surfaces(): router._count_pre_call_check_tokens(messages=None, input=None) +def test_count_pre_call_check_tokens_skips_the_history_a_compaction_item_replaces(): + router = litellm.Router(model_list=[{"model_name": "m", "litellm_params": {"model": "gpt-3.5-turbo"}}]) + summary: Final = "The user planned a Tokyo trip." + compaction_item: Final = { + "type": "compaction", + "id": "cmp_1", + "encrypted_content": json.dumps({"type": "compaction", "content": summary}), + } + history: Final = [ + {"role": "user", "content": "Plan a Tokyo trip. " * 200}, + {"role": "assistant", "content": "Day one, Asakusa. " * 200}, + ] + next_turn: Final = {"role": "user", "content": "And Osaka?"} + + compacted_tokens: Final = router._count_pre_call_check_tokens( + messages=None, input=[*history, compaction_item, next_turn] + ) + + assert compacted_tokens == router._count_pre_call_check_tokens( + messages=None, input=[{"role": "user", "content": summary}, next_turn] + ) + + def test_pre_call_checks_no_messages_or_input_does_not_crash(monkeypatch): """ When neither messages nor input is provided (e.g. endpoints without prompt text), diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 5a2cefd08b1..b0086662f1c 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -254,6 +254,7 @@ INTERNAL_STATE_NAMES: Final = ( "_context_compaction_state", *AGENTIC_LOOP_STATE_NAMES, "_router_weights", + "_encrypted_content_affinity_pinned", "fallback_depth", "max_fallbacks", "attempted_targets", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2b174a1141a..bd08b743c7d 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -44279,7 +44279,7 @@ export interface components { /** Output */ output: (components["schemas"]["ResponseOutputMessage"] | components["schemas"]["ResponseFileSearchToolCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionToolCallOutputItem"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["ResponseComputerToolCall"] | components["schemas"]["ResponseComputerToolCallOutputItem"] | components["schemas"]["ResponseReasoningItem"] | components["schemas"]["ResponseToolSearchCall"] | components["schemas"]["ResponseToolSearchOutputItem"] | components["schemas"]["ResponseCompactionItem"] | components["schemas"]["ImageGenerationCall"] | components["schemas"]["ResponseCodeInterpreterToolCall"] | components["schemas"]["LocalShellCall"] | components["schemas"]["LocalShellCallOutput"] | components["schemas"]["ResponseFunctionShellToolCall"] | components["schemas"]["ResponseFunctionShellToolCallOutput"] | components["schemas"]["ResponseApplyPatchToolCall"] | components["schemas"]["ResponseApplyPatchToolCallOutput"] | components["schemas"]["McpCall"] | components["schemas"]["McpListTools"] | components["schemas"]["McpApprovalRequest"] | components["schemas"]["McpApprovalResponse"] | components["schemas"]["ResponseCustomToolCall"] | components["schemas"]["ResponseCustomToolCallOutputItem"] | { [key: string]: unknown; - })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["OutputToolSearchCall"])[]; + })[] | (components["schemas"]["GenericResponseOutputItem"] | components["schemas"]["OutputCodeInterpreterCall"] | components["schemas"]["OutputFunctionToolCall"] | components["schemas"]["OutputImageGenerationCall"] | components["schemas"]["ResponseFunctionToolCall"] | components["schemas"]["ResponseFunctionWebSearch"] | components["schemas"]["CustomToolCallOutputItem"] | components["schemas"]["OutputToolSearchCall"] | components["schemas"]["ResponseCompactionItem"])[]; /** Parallel Tool Calls */ parallel_tool_calls?: boolean | null; /** Previous Response Id */ From a31c5230de303d342f62e0e2ecd62af83e5fcd4e Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:42:05 +0000 Subject: [PATCH 060/235] fix(proxy): forward dispatched_call_type through _failure_usage_to_lift (#47815) --- litellm/proxy/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 52b3866d04d..28be11509ae 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1294,6 +1294,7 @@ def _failure_usage_to_lift( provider_billed: bool = False, deployment_id: str | None = None, base_model: str | None = None, + dispatched_call_type: str | None = None, ) -> tuple[object, object] | None: current_attempt: Final = _current_failure_usage_to_lift( model_call_details, @@ -1302,6 +1303,7 @@ def _failure_usage_to_lift( provider_billed, deployment_id, base_model, + dispatched_call_type=dispatched_call_type, ) failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) if not failed_attempts: From 9d8fe1ddb51c0cb4c3190aa51274fbdde98eb05f Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 21 Oct 2026 11:58:31 +0000 Subject: [PATCH 061/235] refactor(types): replace Any with proven types in 50 files (#47818) * refactor(types): replace Any with proven types in 100 files Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(types): keep untyped parsing where strict validation changed accepted shapes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(types): keep the OpenAI-compatible response parse untyped Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(types): keep only annotation changes, drop new runtime validation Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/integrations/deepeval/types.py | 4 ++-- .../streaming_iterator.py | 6 +++--- litellm/llms/base_llm/ocr/transformation.py | 6 +++--- litellm/llms/base_llm/sandbox/transformation.py | 6 +++--- .../llms/github_copilot/embedding/transformation.py | 4 +++- litellm/llms/ollama/completion/handler.py | 4 ++-- litellm/models/managed_files.py | 8 ++++---- litellm/proxy/a2a/endpoints.py | 6 +++--- litellm/proxy/auth/auth_exception_handler.py | 2 +- litellm/proxy/management_endpoints/types.py | 4 ++-- litellm/proxy/spend_tracking/vantage_endpoints.py | 4 +++- litellm/rust_bridge/detectors.py | 2 +- litellm/types/caching.py | 4 ++-- litellm/types/completion.py | 2 +- litellm/types/extraction.py | 3 ++- litellm/types/google_genai/main.py | 2 +- litellm/types/images/main.py | 4 ++-- litellm/types/integrations/argilla.py | 4 ++-- litellm/types/integrations/cloudzero.py | 7 ++----- .../types/integrations/compression_interception.py | 4 ++-- litellm/types/integrations/gcs_bucket.py | 4 ++-- litellm/types/integrations/langsmith.py | 8 ++++---- litellm/types/integrations/posthog.py | 4 ++-- .../llms/anthropic_messages/anthropic_response.py | 4 ++-- litellm/types/llms/base.py | 4 ++-- litellm/types/llms/databricks.py | 6 +++--- .../management_endpoints/cache_settings_endpoints.py | 6 +++--- .../router_settings_endpoints.py | 6 +++--- litellm/types/mcp.py | 10 +++++----- litellm/types/mcp_server/mcp_server_manager.py | 4 ++-- litellm/types/prompts/init_prompts.py | 6 +++--- litellm/types/proxy/callback_logs_endpoints.py | 4 ++-- litellm/types/proxy/cloudzero_endpoints.py | 2 +- .../guardrail_hooks/azure/azure_prompt_shield.py | 4 +--- .../guardrail_hooks/azure/azure_text_moderation.py | 4 ++-- .../proxy/guardrails/guardrail_hooks/compresr.py | 4 ++-- .../guardrail_hooks/litellm_content_filter.py | 6 +++--- .../proxy/guardrails/guardrail_hooks/presidio.py | 8 ++++---- .../management_endpoints/common_daily_activity.py | 5 ++--- .../proxy/management_endpoints/config_overrides.py | 6 ++---- .../management_endpoints/internal_user_endpoints.py | 4 ++-- .../model_management_endpoints.py | 3 +-- .../proxy/management_endpoints/team_endpoints.py | 8 ++++---- litellm/types/proxy/prompt_endpoints.py | 4 +--- .../types/proxy/public_endpoints/public_endpoints.py | 4 ++-- litellm/types/proxy/vantage_endpoints.py | 6 +++--- litellm/types/rag.py | 6 +++--- litellm/types/responses/main.py | 4 ++-- litellm/types/vector_store_files.py | 12 +++++++----- litellm/types/vector_stores.py | 2 +- 50 files changed, 120 insertions(+), 124 deletions(-) diff --git a/litellm/integrations/deepeval/types.py b/litellm/integrations/deepeval/types.py index eb4bb1f9b2a..a547863e26a 100644 --- a/litellm/integrations/deepeval/types.py +++ b/litellm/integrations/deepeval/types.py @@ -1,6 +1,6 @@ # Duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/tracing/api.py from enum import Enum -from typing import Any, ClassVar, Literal +from typing import ClassVar, Literal from pydantic import ConfigDict, Field @@ -55,7 +55,7 @@ class TraceApi(LiteLLMBaseModel): tool_spans: list[BaseApiSpan] = Field(alias="toolSpans") start_time: str = Field(alias="startTime") end_time: str = Field(alias="endTime") - metadata: dict[str, Any] | None = Field(None) + metadata: dict[str, object] | None = Field(None) tags: list[str] | None = Field(None) environment: str | None = Field(None) diff --git a/litellm/interactions/litellm_responses_transformation/streaming_iterator.py b/litellm/interactions/litellm_responses_transformation/streaming_iterator.py index dab12d447df..46e633d2b96 100644 --- a/litellm/interactions/litellm_responses_transformation/streaming_iterator.py +++ b/litellm/interactions/litellm_responses_transformation/streaming_iterator.py @@ -3,8 +3,8 @@ Streaming iterator for transforming Responses API stream to Interactions API str """ from collections import deque -from collections.abc import AsyncIterator, Iterator -from typing import Any, Final, cast +from collections.abc import AsyncIterator, Iterator, Mapping +from typing import Final, cast from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, @@ -44,7 +44,7 @@ class LiteLLMResponsesInteractionsStreamingIterator: request_input: InteractionInput | None, optional_params: InteractionsAPIOptionalRequestParams, custom_llm_provider: str | None = None, - litellm_metadata: dict[str, Any] | None = None, + litellm_metadata: Mapping[str, object] | None = None, ): self.model = model self.responses_stream_iterator = litellm_custom_stream_wrapper diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index 97d402e7a1b..171b193f60c 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -4,7 +4,7 @@ Base OCR types shared by the Rust OCR route and Python consumers. import builtins from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import PrivateAttr @@ -45,7 +45,7 @@ class OCRPageImage(LiteLLMPydanticObjectBase): """Image extracted from OCR page.""" image_base64: str | None = None - bbox: dict[str, Any] | None = None + bbox: dict[str, builtins.object] | None = None model_config = {"extra": "allow"} @@ -80,7 +80,7 @@ class OCRResponse(LiteLLMPydanticObjectBase): pages: list[OCRPage] model: str - document_annotation: Any | None = None + document_annotation: builtins.object | None = None usage_info: OCRUsageInfo | None = None content: str | None = None tables: list[dict[str, builtins.object]] | None = None diff --git a/litellm/llms/base_llm/sandbox/transformation.py b/litellm/llms/base_llm/sandbox/transformation.py index 877edc15ec7..fe0e3abb87e 100644 --- a/litellm/llms/base_llm/sandbox/transformation.py +++ b/litellm/llms/base_llm/sandbox/transformation.py @@ -7,7 +7,7 @@ run code -> delete container; `code_interpreter_tool` combines all three. """ import builtins -from typing import Any, Final +from typing import Final import httpx from pydantic import Field, PrivateAttr @@ -42,8 +42,8 @@ class CodeExecutionResult(LiteLLMPydanticObjectBase): stdout: str = "" stderr: str = "" - results: list[dict[str, Any]] = Field(default_factory=list) - error: dict[str, Any] | None = None + results: list[dict[str, object]] = Field(default_factory=list) + error: dict[str, object] | None = None execution_count: int | None = None object: str = "code_execution" diff --git a/litellm/llms/github_copilot/embedding/transformation.py b/litellm/llms/github_copilot/embedding/transformation.py index f16564ba505..4f308b01159 100644 --- a/litellm/llms/github_copilot/embedding/transformation.py +++ b/litellm/llms/github_copilot/embedding/transformation.py @@ -202,7 +202,9 @@ class GithubCopilotEmbeddingConfig(BaseEmbeddingConfig): optional_params[param] = value return optional_params - def get_error_class(self, error_message: str, status_code: int, headers: Any) -> BaseLLMException: + def get_error_class( + self, error_message: str, status_code: int, headers: dict[str, str] | httpx.Headers + ) -> BaseLLMException: from litellm.llms.openai.openai import OpenAIConfig return OpenAIConfig().get_error_class(error_message=error_message, status_code=status_code, headers=headers) diff --git a/litellm/llms/ollama/completion/handler.py b/litellm/llms/ollama/completion/handler.py index d4110e55bf0..0c13b1cd1d3 100644 --- a/litellm/llms/ollama/completion/handler.py +++ b/litellm/llms/ollama/completion/handler.py @@ -87,7 +87,7 @@ async def ollama_aembeddings( model: str, prompts: list[str], model_response: EmbeddingResponse, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, encoding: TokenEncoder | None, ): @@ -110,7 +110,7 @@ def ollama_embeddings( api_base: str, model: str, prompts: list[str], - optional_params: dict, + optional_params: Mapping[str, object], model_response: EmbeddingResponse, logging_obj: object, encoding: TokenEncoder | None = None, diff --git a/litellm/models/managed_files.py b/litellm/models/managed_files.py index 435db632026..b2bfb572b7d 100644 --- a/litellm/models/managed_files.py +++ b/litellm/models/managed_files.py @@ -6,7 +6,7 @@ Canonical definitions for the ``litellm_managed*`` tables. Re-exported from """ from datetime import datetime -from typing import Any, Literal +from typing import Literal from litellm.types.llms.base import LiteLLMPydanticObjectBase from litellm.types.llms.openai import OpenAIFileObject, ResponsesAPIResponse @@ -39,7 +39,7 @@ class LiteLLM_ManagedVectorStoreTable(LiteLLMPydanticObjectBase): """Table for managing vector stores with target_model_names support.""" unified_resource_id: str - resource_object: Any | None = None + resource_object: object | None = None model_mappings: dict[str, str] flat_model_resource_ids: list[str] created_by: str | None = None @@ -54,11 +54,11 @@ class LiteLLM_ManagedVectorStoresTable(LiteLLMPydanticObjectBase): custom_llm_provider: str vector_store_name: str | None = None vector_store_description: str | None = None - vector_store_metadata: dict[str, Any] | None = None + vector_store_metadata: dict[str, object] | None = None created_at: datetime | None = None updated_at: datetime | None = None litellm_credential_name: str | None = None - litellm_params: dict[str, Any] | None = None + litellm_params: dict[str, object] | None = None team_id: str | None = None user_id: str | None = None is_config: bool = False diff --git a/litellm/proxy/a2a/endpoints.py b/litellm/proxy/a2a/endpoints.py index 56b512df934..730d40b46d0 100644 --- a/litellm/proxy/a2a/endpoints.py +++ b/litellm/proxy/a2a/endpoints.py @@ -9,7 +9,7 @@ admin pick which ones to expose through the proxy. The actual merge into a LiteLLM-fronted card happens when the agent is saved via ``POST /v1/agents``. """ -from typing import Any, Final +from typing import Final from fastapi import APIRouter, Depends, HTTPException from fastapi.responses import JSONResponse @@ -50,7 +50,7 @@ class DiscoverAgentRequest(LiteLLMBaseModel): "query parameter." ), ) - params: dict[str, Any] | None = Field( + params: dict[str, object] | None = Field( default=None, description=( "Mode-specific parameters. ``langgraph_platform`` requires " @@ -61,7 +61,7 @@ class DiscoverAgentRequest(LiteLLMBaseModel): class DiscoverAgentResponse(LiteLLMBaseModel): url: str - agent_card: dict[str, Any] + agent_card: dict[str, object] @router.post( diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index ae473a62271..f6637a1fdf3 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -41,7 +41,7 @@ DB_UNAVAILABLE_FALLBACK_USER_ID: Final = "__db_unavailable_fallback__" if TYPE_CHECKING: from opentelemetry.trace import Span as _Span - Span = _Span | Any + Span = _Span else: Span = Any diff --git a/litellm/proxy/management_endpoints/types.py b/litellm/proxy/management_endpoints/types.py index 4414eed97b2..221a6221fab 100644 --- a/litellm/proxy/management_endpoints/types.py +++ b/litellm/proxy/management_endpoints/types.py @@ -5,7 +5,7 @@ Might include fastapi/proxy requirements.txt related imports """ from collections.abc import Iterable, Sequence -from typing import Any, Final, cast +from typing import Final, cast from fastapi_sso.sso.base import OpenID @@ -94,4 +94,4 @@ def get_litellm_user_role(role_str: object) -> LitellmUserRoles | None: class CustomOpenID(OpenID): team_ids: list[str] user_role: LitellmUserRoles | None = None - extra_fields: dict[str, Any] | None = None + extra_fields: dict[str, object] | None = None diff --git a/litellm/proxy/spend_tracking/vantage_endpoints.py b/litellm/proxy/spend_tracking/vantage_endpoints.py index d3858f95243..6938588444c 100644 --- a/litellm/proxy/spend_tracking/vantage_endpoints.py +++ b/litellm/proxy/spend_tracking/vantage_endpoints.py @@ -122,7 +122,9 @@ async def _get_vantage_settings(): encrypted_api_key: Final = settings.get("api_key") if encrypted_api_key: - decrypted_api_key = decrypt_value_helper(encrypted_api_key, key="vantage_api_key", exception_type="error") + decrypted_api_key: Final = decrypt_value_helper( + encrypted_api_key, key="vantage_api_key", exception_type="error" + ) if decrypted_api_key is None: raise HTTPException( status_code=500, diff --git a/litellm/rust_bridge/detectors.py b/litellm/rust_bridge/detectors.py index 0ad1b787e38..0af3a2e6fa6 100644 --- a/litellm/rust_bridge/detectors.py +++ b/litellm/rust_bridge/detectors.py @@ -128,7 +128,7 @@ def _custom_loggers(callbacks: Iterable[object]) -> tuple[CustomLogger, ...]: def overridden(method: str, callbacks: Iterable[object]) -> bool: - base: Final = getattr(CustomLogger, method) + base: Final[object] = getattr(CustomLogger, method) return any(getattr(type(callback), method, base) is not base for callback in _custom_loggers(callbacks)) diff --git a/litellm/types/caching.py b/litellm/types/caching.py index b9bcbbb93dd..ced2065a674 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -75,7 +75,7 @@ class RedisPipelineSetOperation(TypedDict): """ key: str - value: Any + value: object ttl: int | None @@ -133,7 +133,7 @@ class HealthCheckCacheParams(LiteLLMBaseModel): host: str | None = None port: str | int | None = None - redis_kwargs: dict[str, Any] | None = None + redis_kwargs: dict[str, object] | None = None namespace: str | None = None redis_version: str | int | float | None = None diff --git a/litellm/types/completion.py b/litellm/types/completion.py index a6693030e89..0f4bdbd75ce 100644 --- a/litellm/types/completion.py +++ b/litellm/types/completion.py @@ -246,7 +246,7 @@ _CompletionDispatchContext = CompletionDispatchContext _CompletionDispatchResult = Union[ - Coroutine[Any, Any, Union["ModelResponse", "CustomStreamWrapper"]], + Coroutine[object, object, Union["ModelResponse", "CustomStreamWrapper"]], "ModelResponse", "CustomStreamWrapper", ] diff --git a/litellm/types/extraction.py b/litellm/types/extraction.py index 4311e549b9a..562bc41f9de 100644 --- a/litellm/types/extraction.py +++ b/litellm/types/extraction.py @@ -13,6 +13,7 @@ from pydantic import ( model_validator, with_config, ) +from pydantic.fields import FieldInfo from typing_extensions import ReadOnly, Required, Self, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase @@ -30,7 +31,7 @@ def _is_none(value: object) -> bool: return value is None -_OMITTED_WHEN_NONE: Final = Field(exclude_if=_is_none) +_OMITTED_WHEN_NONE: Final = FieldInfo.from_field(exclude_if=_is_none) OptionalJson: TypeAlias = Annotated[JsonValue, _OMITTED_WHEN_NONE] OptionalFloat: TypeAlias = Annotated[float | None, _OMITTED_WHEN_NONE] OptionalInt: TypeAlias = Annotated[int | None, _OMITTED_WHEN_NONE] diff --git a/litellm/types/google_genai/main.py b/litellm/types/google_genai/main.py index 0241c9fa809..dcde0a031e3 100644 --- a/litellm/types/google_genai/main.py +++ b/litellm/types/google_genai/main.py @@ -17,7 +17,7 @@ if TYPE_CHECKING: ToolConfigDict = _genai_types.ToolConfigDict class GenerateContentRequestDict(GenerateContentRequestParametersDict): - generationConfig: Any | None + generationConfig: object | None tools: ToolConfigDict | None class GenerateContentResponse(GoogleGenAIGenerateContentResponse, BaseLiteLLMOpenAIResponseObject): diff --git a/litellm/types/images/main.py b/litellm/types/images/main.py index 5d80135a8a1..963c980a4b8 100644 --- a/litellm/types/images/main.py +++ b/litellm/types/images/main.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal from typing_extensions import TypedDict @@ -20,7 +20,7 @@ class ImageEditOptionalRequestParams(TypedDict, total=False): response_format: Literal["url", "b64_json"] | None size: str | None user: str | None - imageConfig: dict[str, Any] | None + imageConfig: dict[str, object] | None class ImageEditRequestParams(ImageEditOptionalRequestParams, total=False): diff --git a/litellm/types/integrations/argilla.py b/litellm/types/integrations/argilla.py index 2c98486c7a7..754916c2f3a 100644 --- a/litellm/types/integrations/argilla.py +++ b/litellm/types/integrations/argilla.py @@ -1,10 +1,10 @@ -from typing import Any, Final +from typing import Final from typing_extensions import TypedDict class ArgillaItem(TypedDict): - fields: dict[str, Any] + fields: dict[str, object] class ArgillaPayload(TypedDict): diff --git a/litellm/types/integrations/cloudzero.py b/litellm/types/integrations/cloudzero.py index 988d501600f..dcc5d03ccca 100644 --- a/litellm/types/integrations/cloudzero.py +++ b/litellm/types/integrations/cloudzero.py @@ -1,7 +1,4 @@ -from typing import Any - - -class CBFRecord(dict[str, Any]): +class CBFRecord(dict[str, object]): """CloudZero Billing Format (CBF) record structure. This class represents a CBF record that is created from LiteLLM usage data @@ -31,4 +28,4 @@ class CBFRecord(dict[str, Any]): # Type alias for better readability in function signatures -CBFRecordDict = dict[str, Any] +CBFRecordDict = dict[str, object] diff --git a/litellm/types/integrations/compression_interception.py b/litellm/types/integrations/compression_interception.py index a1e66e5057e..e9c40fbf838 100644 --- a/litellm/types/integrations/compression_interception.py +++ b/litellm/types/integrations/compression_interception.py @@ -2,7 +2,7 @@ Type definitions for Compression Interception integration. """ -from typing import Any, Literal, TypedDict +from typing import Literal, TypedDict class CompressionInterceptionConfig(TypedDict, total=False): @@ -24,7 +24,7 @@ class CompressionInterceptionConfig(TypedDict, total=False): compression_trigger: int compression_target: int | None embedding_model: str | None - embedding_model_params: dict[str, Any] | None + embedding_model_params: dict[str, object] | None class CompressionSavingsMetadata(TypedDict): diff --git a/litellm/types/integrations/gcs_bucket.py b/litellm/types/integrations/gcs_bucket.py index 3840d7681a5..79563a0a6e4 100644 --- a/litellm/types/integrations/gcs_bucket.py +++ b/litellm/types/integrations/gcs_bucket.py @@ -31,5 +31,5 @@ class GCSLogQueueItem(TypedDict): """ payload: StandardLoggingPayload - kwargs: dict[str, Any] - response_obj: Any | None + kwargs: dict[str, object] + response_obj: object | None diff --git a/litellm/types/integrations/langsmith.py b/litellm/types/integrations/langsmith.py index 594190ddb5c..f053a5864c1 100644 --- a/litellm/types/integrations/langsmith.py +++ b/litellm/types/integrations/langsmith.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from datetime import datetime -from typing import Any, NamedTuple +from typing import NamedTuple from typing_extensions import TypedDict @@ -9,7 +9,7 @@ from litellm.types.llms.base import LiteLLMBaseModel class LangsmithInputs(LiteLLMBaseModel): model: str | None = None - messages: list[Any] | None = None + messages: list[object] | None = None stream: bool | None = None call_type: str | None = None litellm_call_id: str | None = None @@ -17,9 +17,9 @@ class LangsmithInputs(LiteLLMBaseModel): temperature: float | None = None max_tokens: int | None = None custom_llm_provider: str | None = None - input: list[Any] | None = None + input: list[object] | None = None log_event_type: str | None = None - original_response: Any | None = None + original_response: object | None = None response_cost: float | None = None # LiteLLM Virtual Key specific fields diff --git a/litellm/types/integrations/posthog.py b/litellm/types/integrations/posthog.py index ac04d08107d..d9739df7df7 100644 --- a/litellm/types/integrations/posthog.py +++ b/litellm/types/integrations/posthog.py @@ -1,4 +1,4 @@ -from typing import Any, Final, TypedDict +from typing import Final, TypedDict POSTHOG_MAX_BATCH_SIZE: Final = 100 @@ -7,7 +7,7 @@ class PostHogEventPayload(TypedDict): """PostHog event payload structure""" event: str # "$ai_generation" or "$ai_embedding" - properties: dict[str, Any] + properties: dict[str, object] distinct_id: str diff --git a/litellm/types/llms/anthropic_messages/anthropic_response.py b/litellm/types/llms/anthropic_messages/anthropic_response.py index 03956ae7a54..9ad13243451 100644 --- a/litellm/types/llms/anthropic_messages/anthropic_response.py +++ b/litellm/types/llms/anthropic_messages/anthropic_response.py @@ -1,5 +1,5 @@ from collections.abc import Sequence -from typing import Any, Literal, TypeAlias +from typing import Literal, TypeAlias from typing_extensions import NotRequired, ReadOnly, TypedDict @@ -20,7 +20,7 @@ class AnthropicResponseTextBlock(TypedDict, total=False): Anthropic Response Text Block: https://docs.anthropic.com/en/api/messages """ - citations: list[dict[str, Any]] | None + citations: list[dict[str, object]] | None text: str type: Literal["text"] diff --git a/litellm/types/llms/base.py b/litellm/types/llms/base.py index 47fec914f9c..5a2da071cec 100644 --- a/litellm/types/llms/base.py +++ b/litellm/types/llms/base.py @@ -1,4 +1,4 @@ -from typing import Any, Final +from typing import Final from openai._models import BaseModel as OpenAIObject from pydantic import ConfigDict @@ -45,7 +45,7 @@ class BaseLiteLLMOpenAIResponseObject(LiteLLMBaseModel): class HiddenParams(OpenAIObject): - original_response: str | Any | None = None + original_response: str | object | None = None model_id: str | None = None # used in Router for individual deployments api_base: str | None = None # returns api base used for making completion call _response_ms: float | None = None diff --git a/litellm/types/llms/databricks.py b/litellm/types/llms/databricks.py index c7023e35777..bc229421d91 100644 --- a/litellm/types/llms/databricks.py +++ b/litellm/types/llms/databricks.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal from pydantic import BaseModel from typing_extensions import ( @@ -23,7 +23,7 @@ class GenericStreamingChunk(TypedDict, total=False): class DatabricksTextContent(TypedDict, total=False): type: Literal["text"] text: Required[str] - citations: list[dict[str, Any]] | None + citations: list[dict[str, object]] | None class DatabricksReasoningSummary(TypedDict): @@ -35,7 +35,7 @@ class DatabricksReasoningSummary(TypedDict): class DatabricksReasoningContent(TypedDict, total=False): type: Literal["reasoning"] summary: Required[list[DatabricksReasoningSummary]] - citations: list[dict[str, Any]] | None + citations: list[dict[str, object]] | None AllDatabricksContentListValues = DatabricksTextContent | DatabricksReasoningContent diff --git a/litellm/types/management_endpoints/cache_settings_endpoints.py b/litellm/types/management_endpoints/cache_settings_endpoints.py index f6394f52c99..3584f8ce1da 100644 --- a/litellm/types/management_endpoints/cache_settings_endpoints.py +++ b/litellm/types/management_endpoints/cache_settings_endpoints.py @@ -2,7 +2,7 @@ Types and field definitions for cache settings management endpoints """ -from typing import Any, Final +from typing import Final from litellm.types.llms.base import LiteLLMBaseModel @@ -10,9 +10,9 @@ from litellm.types.llms.base import LiteLLMBaseModel class CacheSettingsField(LiteLLMBaseModel): field_name: str field_type: str - field_value: Any + field_value: object field_description: str - field_default: Any = None + field_default: object = None options: list[str] | None = None # For fields with predefined options/enum values ui_field_name: str # User-friendly display name link: str | None = None # Documentation link for the field diff --git a/litellm/types/management_endpoints/router_settings_endpoints.py b/litellm/types/management_endpoints/router_settings_endpoints.py index 9c8fe446d72..64068f30a16 100644 --- a/litellm/types/management_endpoints/router_settings_endpoints.py +++ b/litellm/types/management_endpoints/router_settings_endpoints.py @@ -2,7 +2,7 @@ Types and field definitions for router settings management endpoints """ -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import Field, field_validator @@ -72,9 +72,9 @@ class FallbackDeleteResponse(LiteLLMBaseModel): class RouterSettingsField(LiteLLMBaseModel): field_name: str field_type: str - field_value: Any + field_value: object field_description: str - field_default: Any = None + field_default: object = None options: list[str] | None = None # For fields with predefined options/enum values ui_field_name: str # User-friendly display name link: str | None = None # Documentation link for the field diff --git a/litellm/types/mcp.py b/litellm/types/mcp.py index f0be66f055d..3fa9adef913 100644 --- a/litellm/types/mcp.py +++ b/litellm/types/mcp.py @@ -113,7 +113,7 @@ class MCPPublicServer(LiteLLMBaseModel): transport: MCPTransportType spec_path: str | None = None auth_type: MCPAuthType | None = None - mcp_info: dict[str, Any] | None = None + mcp_info: dict[str, object] | None = None class MCPAllowedClient(LiteLLMBaseModel): @@ -448,11 +448,11 @@ class MCPPreCallRequestObject(LiteLLMBaseModel): """ tool_name: str - arguments: dict[str, Any] + arguments: dict[str, object] server_name: str | None = None tool_description: str | None = None tool_input_schema: Mapping[str, object] | None = None - user_api_key_auth: dict[str, Any] | None = None + user_api_key_auth: dict[str, object] | None = None hidden_params: HiddenParams = HiddenParams() @@ -462,7 +462,7 @@ class MCPPreCallResponseObject(LiteLLMBaseModel): """ should_proceed: bool = True - modified_arguments: dict[str, Any] | None = None + modified_arguments: dict[str, object] | None = None error_message: str | None = None hidden_params: HiddenParams = HiddenParams() @@ -473,7 +473,7 @@ class MCPDuringCallRequestObject(LiteLLMBaseModel): """ tool_name: str - arguments: dict[str, Any] + arguments: dict[str, object] server_name: str | None = None tool_description: str | None = None tool_input_schema: Mapping[str, object] | None = None diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 219f23200d2..d29e4db55a4 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -117,7 +117,7 @@ class MCPServer(LiteLLMBaseModel): # Admin-configured env vars. Each entry is {name, value, scope, description}. # scope=="global" values are interpolated into static_headers using ${NAME}. # scope=="user" values must be supplied per-user. - env_vars: list[dict[str, Any]] | None = None + env_vars: list[dict[str, object]] | None = None # OAuth-specific fields authorization_server_metadata: Mapping[str, object] | None = Field(default=None, exclude=True) client_id: str | None = None @@ -240,7 +240,7 @@ class MCPServer(LiteLLMBaseModel): # token_validation: key-value pairs that must match fields in the OAuth token # response (supports dot-notation for nested fields, e.g. "team.enterprise_id"). # Tokens that fail validation are rejected before storage. - token_validation: dict[str, Any] | None = None + token_validation: dict[str, object] | None = None oauth_identity_binding: MCPOAuthIdentityBinding | None = None # Optional TTL override (seconds) for the Redis per-user token cache, capped # at the token's expires_in minus the expiry buffer so a cached entry never diff --git a/litellm/types/prompts/init_prompts.py b/litellm/types/prompts/init_prompts.py index fc4c36e760a..c1a3ab56311 100644 --- a/litellm/types/prompts/init_prompts.py +++ b/litellm/types/prompts/init_prompts.py @@ -1,6 +1,6 @@ from datetime import datetime from enum import Enum -from typing import Any, Literal +from typing import Literal from pydantic import ConfigDict @@ -31,7 +31,7 @@ class PromptLiteLLMParams(LiteLLMBaseModel): api_base: str | None = None api_key: str | None = None - provider_specific_query_params: dict[str, Any] | None = None + provider_specific_query_params: dict[str, object] | None = None ignore_prompt_manager_model: bool | None = False ignore_prompt_manager_optional_params: bool | None = False @@ -66,7 +66,7 @@ class PromptSpec(LiteLLMBaseModel): class PromptTemplateBase(LiteLLMBaseModel): litellm_prompt_id: str content: str - metadata: dict[str, Any] | None = None + metadata: dict[str, object] | None = None class PromptInfoResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/callback_logs_endpoints.py b/litellm/types/proxy/callback_logs_endpoints.py index b2519d97122..0291cc902bc 100644 --- a/litellm/types/proxy/callback_logs_endpoints.py +++ b/litellm/types/proxy/callback_logs_endpoints.py @@ -5,7 +5,7 @@ External producers (e.g. the litellm-rust gateway) POST finished logging payloads here; the proxy replays them through the standard callback fan-out. """ -from typing import Any, Literal +from typing import Literal from pydantic import Field @@ -17,7 +17,7 @@ class CallbackLogRecord(LiteLLMBaseModel): """A single finished logging event to replay through the callbacks.""" status: Literal["success", "failure"] - standard_logging_payload: dict[str, Any] + standard_logging_payload: dict[str, object] error: str | None = None diff --git a/litellm/types/proxy/cloudzero_endpoints.py b/litellm/types/proxy/cloudzero_endpoints.py index ece294674b9..f87537494bd 100644 --- a/litellm/types/proxy/cloudzero_endpoints.py +++ b/litellm/types/proxy/cloudzero_endpoints.py @@ -46,7 +46,7 @@ class CloudZeroExportResponse(LiteLLMBaseModel): dry_run_data: dict[str, Any] | None = Field( None, description="Dry run data including usage data and CBF transformed data" ) - summary: dict[str, Any] | None = Field(None, description="Summary statistics for dry run") + summary: dict[str, object] | None = Field(None, description="Summary statistics for dry run") class CloudZeroSettingsView(LiteLLMBaseModel): diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_prompt_shield.py b/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_prompt_shield.py index 60846b2a1bd..4f4bebf9704 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_prompt_shield.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_prompt_shield.py @@ -1,5 +1,3 @@ -from typing import Any - from pydantic import Field from typing_extensions import TypedDict @@ -23,7 +21,7 @@ class AzurePromptShieldGuardrailResponse(TypedDict): """Configuration parameters for the Azure Prompt Shield guardrail""" userPromptAnalysis: UserPromptAnalysis - documentsAnalysis: list[dict[str, Any]] + documentsAnalysis: list[dict[str, object]] class AzurePromptShieldGuardrailConfigModel( diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_text_moderation.py b/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_text_moderation.py index 90662e84ed5..6ee5741baf5 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_text_moderation.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/azure/azure_text_moderation.py @@ -1,4 +1,4 @@ -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import Field from typing_extensions import Required, TypedDict @@ -37,7 +37,7 @@ class AzureTextModerationGuardrailResponseCategoriesAnalysis(TypedDict): class AzureTextModerationGuardrailResponse(TypedDict): """Response from the Azure Text Moderation guardrail""" - blocklistsMatch: list[dict[str, Any]] + blocklistsMatch: list[dict[str, object]] categoriesAnalysis: list[AzureTextModerationGuardrailResponseCategoriesAnalysis] diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/compresr.py b/litellm/types/proxy/guardrails/guardrail_hooks/compresr.py index 6598f1de62b..c0f5ea5346e 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/compresr.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/compresr.py @@ -1,4 +1,4 @@ -from typing import Any, Literal +from typing import Literal from pydantic import Field @@ -93,7 +93,7 @@ class CompresrGuardrailOptionalParams(LiteLLMBaseModel): "Unset lets the server default apply (~10.0)." ), ) - compression_params: dict[str, Any] | None = Field( + compression_params: dict[str, object] | None = Field( default=None, description=( "Passthrough of extra parameters forwarded verbatim in the Compresr " diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/litellm_content_filter.py b/litellm/types/proxy/guardrails/guardrail_hooks/litellm_content_filter.py index 78800a5e9a2..d49a3afae04 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/litellm_content_filter.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/litellm_content_filter.py @@ -1,5 +1,5 @@ from enum import Enum -from typing import Any, Literal, TypedDict +from typing import Literal, TypedDict from pydantic import Field @@ -77,7 +77,7 @@ class CompetitorIntentDetection(TypedDict): action_hint: str entities: dict[str, list[str]] signals: list[str] - evidence: list[dict[str, Any]] + evidence: list[dict[str, object]] ContentFilterDetection = PatternDetection | BlockedWordDetection | CategoryKeywordDetection | CompetitorIntentDetection @@ -157,7 +157,7 @@ class LitellmContentFilterGuardrailConfigModel(GuardrailConfigModel): ) # Competitor intent blocker (generic; industry presets add domain_words, etc.) - competitor_intent_config: dict[str, Any] | None = Field( + competitor_intent_config: dict[str, object] | None = Field( default=None, description="Optional config for intent-based competitor comparison detection. " "Keys: brand_self (list), competitors (list), competitor_aliases (dict), " diff --git a/litellm/types/proxy/guardrails/guardrail_hooks/presidio.py b/litellm/types/proxy/guardrails/guardrail_hooks/presidio.py index 9d1e375af72..6fe324ce6b0 100644 --- a/litellm/types/proxy/guardrails/guardrail_hooks/presidio.py +++ b/litellm/types/proxy/guardrails/guardrail_hooks/presidio.py @@ -1,6 +1,6 @@ -from typing import Any +from collections.abc import Mapping -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict from litellm.types.guardrails import PiiEntityType @@ -17,5 +17,5 @@ class PresidioAnalyzeResponseItem(TypedDict, total=False): start: int | None end: int | None score: float | None - analysis_explanation: dict[str, Any] | None - recognition_metadata: dict[str, Any] | None + analysis_explanation: ReadOnly[Mapping[str, object] | None] + recognition_metadata: ReadOnly[Mapping[str, object] | None] diff --git a/litellm/types/proxy/management_endpoints/common_daily_activity.py b/litellm/types/proxy/management_endpoints/common_daily_activity.py index 66702abfb34..213c911cfdc 100644 --- a/litellm/types/proxy/management_endpoints/common_daily_activity.py +++ b/litellm/types/proxy/management_endpoints/common_daily_activity.py @@ -1,7 +1,6 @@ -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from datetime import date from enum import Enum -from typing import Any from pydantic import ConfigDict, Field from typing_extensions import TypedDict @@ -61,7 +60,7 @@ class KeyMetricWithMetadata(MetricBase): class MetricWithMetadata(MetricBase): - metadata: dict[str, Any] = Field(default_factory=dict) + metadata: Mapping[str, object] = Field(default_factory=dict) # API key breakdown for this metric (e.g., which API keys are using this MCP server) api_key_breakdown: dict[str, KeyMetricWithMetadata] = Field(default_factory=dict) # api_key -> {metrics, metadata} diff --git a/litellm/types/proxy/management_endpoints/config_overrides.py b/litellm/types/proxy/management_endpoints/config_overrides.py index c82e616cf9a..b807b5537d5 100644 --- a/litellm/types/proxy/management_endpoints/config_overrides.py +++ b/litellm/types/proxy/management_endpoints/config_overrides.py @@ -1,5 +1,3 @@ -from typing import Any - from pydantic import Field from litellm.types.llms.base import LiteLLMBaseModel @@ -103,5 +101,5 @@ class ConfigOverrideSettingsResponse(LiteLLMBaseModel): """Response model for config override settings GET endpoints.""" config_type: str = Field(description="The type of config override") - values: dict[str, Any] = Field(description="Current configuration values (sensitive fields decrypted)") - field_schema: dict[str, Any] = Field(description="Schema information for UI rendering") + values: dict[str, object] = Field(description="Current configuration values (sensitive fields decrypted)") + field_schema: dict[str, object] = Field(description="Schema information for UI rendering") diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index febd4485d10..fc0b73f77b6 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import ConfigDict, Field, field_validator from typing_extensions import ReadOnly, TypedDict @@ -80,7 +80,7 @@ class UserUpdateResult(LiteLLMBaseModel): user_email: str | None = None success: bool error: str | None = None - updated_user: dict[str, Any] | None = None + updated_user: dict[str, object] | None = None class BulkUpdateUserResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/management_endpoints/model_management_endpoints.py b/litellm/types/proxy/management_endpoints/model_management_endpoints.py index 6a4160c3c50..c012f872280 100644 --- a/litellm/types/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/model_management_endpoints.py @@ -1,6 +1,5 @@ from collections.abc import Mapping from datetime import datetime -from typing import Any from pydantic import ConfigDict, Field @@ -22,7 +21,7 @@ class UpdateUsefulLinksRequest(LiteLLMBaseModel): # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) - useful_links: dict[str, str | dict[str, Any]] + useful_links: dict[str, str | dict[str, object]] class AutoRouterClassifierDefaultPromptResponse(LiteLLMBaseModel): diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 743b2d7c963..51c48e6ebf9 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,5 +1,5 @@ from collections.abc import Mapping -from typing import Any, Final, Literal +from typing import Final, Literal from pydantic import ConfigDict, Field, field_validator, model_validator @@ -121,8 +121,8 @@ class TeamMemberAddResult(LiteLLMBaseModel): user_email: str | None = None success: bool error: str | None = None - updated_user: dict[str, Any] | None = None - updated_team_membership: dict[str, Any] | None = None + updated_user: dict[str, object] | None = None + updated_team_membership: dict[str, object] | None = None class BulkTeamMemberAddResponse(LiteLLMBaseModel): @@ -133,7 +133,7 @@ class BulkTeamMemberAddResponse(LiteLLMBaseModel): total_requested: int successful_additions: int failed_additions: int - updated_team: dict[str, Any] | None = None + updated_team: dict[str, object] | None = None class TeamMemberRef(MemberDeleteRequest): diff --git a/litellm/types/proxy/prompt_endpoints.py b/litellm/types/proxy/prompt_endpoints.py index 930713663a1..cded9f5fb87 100644 --- a/litellm/types/proxy/prompt_endpoints.py +++ b/litellm/types/proxy/prompt_endpoints.py @@ -1,9 +1,7 @@ -from typing import Any - from litellm.types.llms.base import LiteLLMBaseModel class TestPromptRequest(LiteLLMBaseModel): dotprompt_content: str - prompt_variables: dict[str, Any] | None = None + prompt_variables: dict[str, object] | None = None conversation_history: list[dict[str, str]] | None = None diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index c487ce8b6c5..0e8a2bb3401 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from typing import Any, Literal +from typing import Literal from pydantic import ConfigDict @@ -13,7 +13,7 @@ class PublicModelHubInfo(LiteLLMBaseModel): # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) # New format: { "displayName": { "url": "...", "index": 0 } } # Old format: { "displayName": "url" } (for backward compatibility) - useful_links: dict[str, str | dict[str, Any]] | None + useful_links: dict[str, str | dict[str, object]] | None class ProviderCredentialField(LiteLLMBaseModel): diff --git a/litellm/types/proxy/vantage_endpoints.py b/litellm/types/proxy/vantage_endpoints.py index 1833d8b8386..ccf050a8921 100644 --- a/litellm/types/proxy/vantage_endpoints.py +++ b/litellm/types/proxy/vantage_endpoints.py @@ -2,8 +2,8 @@ Vantage endpoint types for LiteLLM Proxy """ +from collections.abc import Mapping from datetime import datetime -from typing import Any from pydantic import Field, field_validator @@ -57,10 +57,10 @@ class VantageExportResponse(LiteLLMBaseModel): message: str status: str - dry_run_data: dict[str, Any] | None = Field( + dry_run_data: Mapping[str, object] | None = Field( None, description="Dry run data including usage data and FOCUS transformed data" ) - summary: dict[str, Any] | None = Field(None, description="Summary statistics for dry run") + summary: Mapping[str, object] | None = Field(None, description="Summary statistics for dry run") class VantageSettingsView(LiteLLMBaseModel): diff --git a/litellm/types/rag.py b/litellm/types/rag.py index 2f912d21587..02c06f242d1 100644 --- a/litellm/types/rag.py +++ b/litellm/types/rag.py @@ -3,7 +3,7 @@ Type definitions for RAG (Retrieval Augmented Generation) Ingest API. """ from collections.abc import Mapping -from typing import Any, Literal +from typing import Literal from pydantic import ConfigDict from typing_extensions import ReadOnly, TypedDict @@ -232,7 +232,7 @@ class RAGIngestRequest(LiteLLMBaseModel): file_url: str | None = None # URL to fetch file from file_id: str | None = None # Existing file ID - ingest_options: dict[str, Any] # RAGIngestOptions as dict for flexibility + ingest_options: dict[str, object] # RAGIngestOptions as dict for flexibility model_config = ConfigDict(extra="allow") # Allow additional fields @@ -260,7 +260,7 @@ class RAGQueryRequest(LiteLLMBaseModel): """Request body for RAG query API.""" model: str - messages: list[Any] + messages: list[object] retrieval_config: RAGRetrievalConfig rerank: RAGRerankConfig | None = None stream: bool | None = False diff --git a/litellm/types/responses/main.py b/litellm/types/responses/main.py index 5588527c141..72dc63a581f 100644 --- a/litellm/types/responses/main.py +++ b/litellm/types/responses/main.py @@ -5,7 +5,7 @@ from typing import Final, Literal, Optional, Union from openai.types.responses.response_function_tool_call import ResponseFunctionToolCall from openai.types.responses.response_function_web_search import ActionSearchSource, ResponseFunctionWebSearch from pydantic import ConfigDict, PrivateAttr -from typing_extensions import Any, TypedDict +from typing_extensions import TypedDict from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject @@ -100,7 +100,7 @@ class OutputCodeInterpreterCall(BaseLiteLLMOpenAIResponseObject): def build_code_interpreter_log_outputs( - content: Any, + content: object, ) -> list[OutputCodeInterpreterCallLog] | None: """Convert Anthropic bash_code_execution stdout/stderr to log outputs. diff --git a/litellm/types/vector_store_files.py b/litellm/types/vector_store_files.py index 6b4953965d5..3623746216a 100644 --- a/litellm/types/vector_store_files.py +++ b/litellm/types/vector_store_files.py @@ -1,7 +1,9 @@ +import builtins +from collections.abc import Mapping from enum import Enum -from typing import Any, Literal +from typing import Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class VectorStoreFileStatus(str, Enum): @@ -33,7 +35,7 @@ class VectorStoreFileObject(TypedDict, total=False): usage_bytes: int | None vector_store_id: str status: VectorStoreFileStatus - last_error: dict[str, Any] | None + last_error: ReadOnly[Mapping[str, builtins.object] | None] chunking_strategy: VectorStoreFileChunkingStrategy | None attributes: dict[str, str | int | float | bool] | None @@ -83,5 +85,5 @@ class VectorStoreFileContentResponse(TypedDict, total=False): class VectorStoreFileAuthCredentials(TypedDict, total=False): - headers: dict[str, Any] - query_params: dict[str, Any] + headers: ReadOnly[Mapping[str, object]] + query_params: ReadOnly[Mapping[str, object]] diff --git a/litellm/types/vector_stores.py b/litellm/types/vector_stores.py index 5e7d517ece2..2259339e479 100644 --- a/litellm/types/vector_stores.py +++ b/litellm/types/vector_stores.py @@ -31,7 +31,7 @@ class LiteLLM_ManagedVectorStore(TypedDict, total=False): vector_store_name: str | None vector_store_description: str | None - vector_store_metadata: dict[str, Any] | str | None + vector_store_metadata: dict[str, object] | str | None created_at: datetime | None updated_at: datetime | None From 3f93350411ddb50b1beb413665df4d0eb5a065de Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 21 Oct 2026 06:37:24 -0700 Subject: [PATCH 062/235] docs: rename security.md to SECURITY.md and point contact to security@berri.ai (#47581) Co-authored-by: krrish-berri-2 <270687000+krrish-berri-2@users.noreply.github.com> --- security.md => SECURITY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename security.md => SECURITY.md (98%) diff --git a/security.md b/SECURITY.md similarity index 98% rename from security.md rename to SECURITY.md index c73379a9c01..d3603448d46 100644 --- a/security.md +++ b/SECURITY.md @@ -79,7 +79,7 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod - Audit Logs with retention policy - Control Allowed IP Addresses that can access your Cloud LiteLLM Instance -For security inquiries, please contact us at support@berri.ai +For security inquiries, please contact us at security@berri.ai #### Supported data regions for LiteLLM Cloud From b6f91a221b6b9df6f8a89ec6f8264ac87d63ff5d Mon Sep 17 00:00:00 2001 From: yujonglee Date: Wed, 21 Oct 2026 07:03:18 -0700 Subject: [PATCH 063/235] refactor(inference): rename RouteError to Error (#47771) Every other crate names its error Error. The format crates already re-exported it under that name, so only the inference crate and a few importers change --- .../crates/gateway-inference/src/caching.rs | 8 +++--- .../crates/gateway-inference/src/error.rs | 26 +++++++++-------- litellm-rust/crates/inference-chat/src/lib.rs | 2 +- .../crates/inference-chat/tests/caching.rs | 10 +++---- .../crates/inference-messages/src/lib.rs | 2 +- .../inference-messages/tests/caching.rs | 10 +++---- litellm-rust/crates/inference-ocr/AGENTS.md | 2 +- .../inference-ocr/src/provider_config.rs | 2 +- .../crates/inference-responses/src/lib.rs | 2 +- .../inference-responses/tests/caching.rs | 10 +++---- .../crates/inference-transcription/src/lib.rs | 2 +- litellm-rust/crates/inference/AGENTS.md | 6 ++-- litellm-rust/crates/inference/src/caching.rs | 26 ++++++++--------- litellm-rust/crates/inference/src/context.rs | 10 +++---- litellm-rust/crates/inference/src/error.rs | 24 ++++++++-------- litellm-rust/crates/inference/src/lib.rs | 2 +- litellm-rust/crates/inference/src/provider.rs | 2 +- .../crates/inference/tests/caching.rs | 24 ++++++++-------- .../crates/python-bridge/src/errors.rs | 28 +++++++++---------- .../python-bridge/src/routes/inference.rs | 10 +++---- 20 files changed, 104 insertions(+), 104 deletions(-) diff --git a/litellm-rust/crates/gateway-inference/src/caching.rs b/litellm-rust/crates/gateway-inference/src/caching.rs index 1abea61b164..426b8b91182 100644 --- a/litellm-rust/crates/gateway-inference/src/caching.rs +++ b/litellm-rust/crates/gateway-inference/src/caching.rs @@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result { #[derive(Clone, Default)] pub(crate) struct CacheHeaders(std::sync::Arc>); -impl litellm_host::interceptors::Interceptors for CacheHeaders { +impl litellm_host::interceptors::Interceptors for CacheHeaders { async fn before_provider_request( &self, wire: litellm_host::interceptors::WireRequest, _: litellm_host::interceptors::RequestContext, - ) -> Result { + ) -> Result { Ok(wire) } async fn after_provider_response( &self, _: litellm_host::interceptors::RawResponse, - ) -> Result<(), litellm_inference::RouteError> { + ) -> Result<(), litellm_inference::Error> { Ok(()) } async fn result_ready( &self, facts: litellm_host::interceptors::ExecutionFacts, - ) -> Result<(), litellm_inference::RouteError> { + ) -> Result<(), litellm_inference::Error> { if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source { let _ = self.0.set(key); } diff --git a/litellm-rust/crates/gateway-inference/src/error.rs b/litellm-rust/crates/gateway-inference/src/error.rs index 39656b933eb..ec82a6ede12 100644 --- a/litellm-rust/crates/gateway-inference/src/error.rs +++ b/litellm-rust/crates/gateway-inference/src/error.rs @@ -4,7 +4,7 @@ use axum::{ response::{IntoResponse, Response}, }; use litellm_http::transport::Error as TransportError; -use litellm_inference::RouteError; +use litellm_inference::Error as InferenceError; use litellm_llms::base_llm::ocr::error::Error as OcrError; use serde_json::{Map, Value, json}; @@ -19,7 +19,7 @@ pub enum Error { )] UnknownModel(String), #[error(transparent)] - Route(#[from] RouteError), + Route(#[from] InferenceError), #[error(transparent)] Ocr(#[from] OcrError), #[error("{0} is not implemented by the Rust gateway")] @@ -36,8 +36,8 @@ impl IntoResponse for Error { } } -impl From> for Error { - fn from(error: litellm_host_http::Error) -> Self { +impl From> for Error { + fn from(error: litellm_host_http::Error) -> Self { match error { litellm_host_http::Error::Call(error) => Self::Route(error), litellm_host_http::Error::Protocol => Self::Internal(error.to_string()), @@ -50,7 +50,7 @@ impl Error { match self { Self::Auth(error) => error.status(), Self::Unsupported(_) - | Self::Route(RouteError::Unsupported(_)) + | Self::Route(InferenceError::Unsupported(_)) | Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED, Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE, Self::Ocr( @@ -64,10 +64,10 @@ impl Error { .and_then(|status| StatusCode::from_u16(status).ok()) .unwrap_or(StatusCode::INTERNAL_SERVER_ERROR), Self::InvalidBody(_) | Self::UnknownModel(_) => StatusCode::BAD_REQUEST, - Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => { + Self::Route(InferenceError::Transport(TransportError::Http { status, .. })) => { StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY) } - Self::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { .. })) => { + Self::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { .. })) => { StatusCode::UNAUTHORIZED } Self::Route(error) if error.is_request() => StatusCode::BAD_REQUEST, @@ -97,7 +97,9 @@ impl Error { /// body already in that shape passes through, any other has its message extracted. pub fn body(&self, request_id: Option<&str>) -> Value { let raw = match self { - Self::Route(RouteError::Transport(TransportError::Http { body, .. })) => body.clone(), + Self::Route(InferenceError::Transport(TransportError::Http { body, .. })) => { + body.clone() + } other => other.to_string(), }; let parsed = serde_json::from_str::(&raw).ok(); @@ -175,7 +177,7 @@ mod tests { use super::*; fn upstream(status: u16, body: &str) -> Error { - Error::Route(RouteError::Transport(TransportError::Http { + Error::Route(InferenceError::Transport(TransportError::Http { status, body: body.into(), })) @@ -225,16 +227,16 @@ mod tests { #[rstest] #[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)] - #[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)] + #[case::rejected_request(Error::Route(InferenceError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)] #[case::missing_key( - Error::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { + Error::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { provider: "Anthropic", environment_variable: "ANTHROPIC_API_KEY", })), StatusCode::UNAUTHORIZED, )] #[case::lost_connection( - Error::Route(RouteError::Transport(TransportError::Network("reset".into()))), + Error::Route(InferenceError::Transport(TransportError::Network("reset".into()))), StatusCode::INTERNAL_SERVER_ERROR, )] fn status_follows_who_is_at_fault(#[case] error: Error, #[case] status: StatusCode) { diff --git a/litellm-rust/crates/inference-chat/src/lib.rs b/litellm-rust/crates/inference-chat/src/lib.rs index f6b391e0ba1..3be83840901 100644 --- a/litellm-rust/crates/inference-chat/src/lib.rs +++ b/litellm-rust/crates/inference-chat/src/lib.rs @@ -1,7 +1,7 @@ use litellm_host::observation::ObservationSender; pub mod route; pub mod types; -pub use litellm_inference::RouteError as Error; +pub use litellm_inference::Error; mod common_utils; pub mod constants; pub(crate) mod handler; diff --git a/litellm-rust/crates/inference-chat/tests/caching.rs b/litellm-rust/crates/inference-chat/tests/caching.rs index 3fc432f8b46..70da8bc3255 100644 --- a/litellm-rust/crates/inference-chat/tests/caching.rs +++ b/litellm-rust/crates/inference-chat/tests/caching.rs @@ -21,7 +21,7 @@ use litellm_host::{ lifecycle::{CallEvent, ExecutionEvent}, observation::observation_channel, }; -use litellm_inference::RouteError; +use litellm_inference::Error; use rstest::{fixture, rstest}; use serde_json::{Value, json}; @@ -166,12 +166,12 @@ struct ChangingHooks { facts: std::sync::Mutex>, } -impl Interceptors for ChangingHooks { +impl Interceptors for ChangingHooks { async fn before_provider_request( &self, mut wire: WireRequest, _: RequestContext, - ) -> Result { + ) -> Result { let call = self.calls.fetch_add(1, Ordering::SeqCst); if self.rewrite { wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 }); @@ -179,11 +179,11 @@ impl Interceptors for ChangingHooks { Ok(wire) } - async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> { + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { Ok(()) } - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { self.facts.lock().unwrap().push(facts); Ok(()) } diff --git a/litellm-rust/crates/inference-messages/src/lib.rs b/litellm-rust/crates/inference-messages/src/lib.rs index f91d70445f3..bfefb9c63b0 100644 --- a/litellm-rust/crates/inference-messages/src/lib.rs +++ b/litellm-rust/crates/inference-messages/src/lib.rs @@ -14,7 +14,7 @@ use std::sync::Arc; use crate::common_utils::messages_provider; -pub use litellm_inference::RouteError as Error; +pub use litellm_inference::Error; pub use types::{ MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params, messages_body, diff --git a/litellm-rust/crates/inference-messages/tests/caching.rs b/litellm-rust/crates/inference-messages/tests/caching.rs index 32c962091ad..480b3c4f3b7 100644 --- a/litellm-rust/crates/inference-messages/tests/caching.rs +++ b/litellm-rust/crates/inference-messages/tests/caching.rs @@ -17,7 +17,7 @@ use litellm_host::interceptors::{ ResultSource, WireRequest, }; use litellm_inference::{ - RouteError, + Error, caching::{CacheRequest, execute_unary}, }; use rstest::{fixture, rstest}; @@ -133,12 +133,12 @@ struct ChangingHooks { facts: std::sync::Mutex>, } -impl Interceptors for ChangingHooks { +impl Interceptors for ChangingHooks { async fn before_provider_request( &self, mut wire: WireRequest, _: RequestContext, - ) -> Result { + ) -> Result { let call = self.calls.fetch_add(1, Ordering::SeqCst); if self.rewrite { wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 }); @@ -146,11 +146,11 @@ impl Interceptors for ChangingHooks { Ok(wire) } - async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> { + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { Ok(()) } - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { self.facts.lock().unwrap().push(facts); Ok(()) } diff --git a/litellm-rust/crates/inference-ocr/AGENTS.md b/litellm-rust/crates/inference-ocr/AGENTS.md index d74b7e7af06..e08f561e75d 100644 --- a/litellm-rust/crates/inference-ocr/AGENTS.md +++ b/litellm-rust/crates/inference-ocr/AGENTS.md @@ -3,4 +3,4 @@ - The route returns its completed response directly - Route closures supply OCR-specific host capabilities, such as the caller's Azure AD token provider (`src/route.rs`) - Provider code reaches the caller's hooks mid-call only through `litellm_llms::base_llm::ocr::handler::CallHooks`, which this crate implements over its host until it folds into `litellm_host::interceptors::Interceptors` -- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::RouteError` until OCR folds into `litellm_llms::Error` +- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::Error` until OCR folds into `litellm_llms::Error` diff --git a/litellm-rust/crates/inference-ocr/src/provider_config.rs b/litellm-rust/crates/inference-ocr/src/provider_config.rs index 2f0346f6e47..09041e9b72b 100644 --- a/litellm-rust/crates/inference-ocr/src/provider_config.rs +++ b/litellm-rust/crates/inference-ocr/src/provider_config.rs @@ -249,7 +249,7 @@ pub(crate) fn resolve_provider_config<'a>( provider: LlmProviders::Mistral, } } - Err(litellm_inference::RouteError::InvalidProvider(provider)) => { + Err(litellm_inference::Error::InvalidProvider(provider)) => { return Err(Error::InvalidProvider(provider)); } Err(error) => return Err(Error::InvalidProvider(error.to_string())), diff --git a/litellm-rust/crates/inference-responses/src/lib.rs b/litellm-rust/crates/inference-responses/src/lib.rs index 6a9d5372311..b6e151ecec3 100644 --- a/litellm-rust/crates/inference-responses/src/lib.rs +++ b/litellm-rust/crates/inference-responses/src/lib.rs @@ -1,5 +1,5 @@ use litellm_host::observation::ObservationSender; -pub use litellm_inference::RouteError as Error; +pub use litellm_inference::Error; pub mod ids; pub mod route; diff --git a/litellm-rust/crates/inference-responses/tests/caching.rs b/litellm-rust/crates/inference-responses/tests/caching.rs index f45cefdc13b..462461695c3 100644 --- a/litellm-rust/crates/inference-responses/tests/caching.rs +++ b/litellm-rust/crates/inference-responses/tests/caching.rs @@ -18,7 +18,7 @@ use litellm_host::interceptors::{ WireRequest, }; use litellm_inference::{ - RouteError, + Error, caching::{CacheRequest, execute_unary}, }; use rstest::{fixture, rstest}; @@ -124,12 +124,12 @@ struct ChangingHooks { facts: std::sync::Mutex>, } -impl Interceptors for ChangingHooks { +impl Interceptors for ChangingHooks { async fn before_provider_request( &self, mut wire: WireRequest, _: RequestContext, - ) -> Result { + ) -> Result { let call = self.calls.fetch_add(1, Ordering::SeqCst); if self.rewrite { wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 }); @@ -137,11 +137,11 @@ impl Interceptors for ChangingHooks { Ok(wire) } - async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> { + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { Ok(()) } - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { self.facts.lock().unwrap().push(facts); Ok(()) } diff --git a/litellm-rust/crates/inference-transcription/src/lib.rs b/litellm-rust/crates/inference-transcription/src/lib.rs index 47e7bc70b31..f11e3abcd54 100644 --- a/litellm-rust/crates/inference-transcription/src/lib.rs +++ b/litellm-rust/crates/inference-transcription/src/lib.rs @@ -1,5 +1,5 @@ pub mod types; -pub use litellm_inference::RouteError as Error; +pub use litellm_inference::Error; mod constants; mod handler; diff --git a/litellm-rust/crates/inference/AGENTS.md b/litellm-rust/crates/inference/AGENTS.md index 2c4bd00108b..7f2bd0387be 100644 --- a/litellm-rust/crates/inference/AGENTS.md +++ b/litellm-rust/crates/inference/AGENTS.md @@ -2,7 +2,7 @@ - `gateway-inference` (serving) -> `inference-` (one API's call) -> `inference` (shared base) - This crate is the shared base every format crate builds on - - `RouteError` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`) + - `Error` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`) - diagnostic spans (`src/diagnostic.rs`), outbound send and signing (`src/outbound.rs`), provider resolution (`src/provider.rs`), `CoreResources` (`src/resources.rs`) - response caching (`src/caching.rs`): `Cachable`, `StreamCachable`, `CacheRequest`, `CallCache`, `execute_unary`, `execute_streaming`, stream capture - Shared test helpers belong in litellm-inference-testing, used only as a dev dependency @@ -49,14 +49,14 @@ - `litellm-inference-` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...) - Provider code never imports from `inference` or `inference-*` - Handlers belong in `inference-*` or `llms`, never in a host crate -- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::RouteError as Error;`, which keeps the pre-split `::Error` name +- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::Error;`, which keeps the pre-split `::Error` name ## Error placement - The workspace `Error definitions` rules shape each crate's error. This section decides which crate and module a failure belongs to - A failure is declared once, by the lowest crate that raises it - every crate above nests it unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error` - - `RouteError` collects route failures for every format crate and never re-declares a variant a lower crate raises + - `Error` collects route failures for every format crate and never re-declares a variant a lower crate raises - Scope follows the concept, not the first caller - an error type under `litellm-llms`'s `/` is private to that provider: no other provider and nothing in `base_llm` may import it - a failure two providers or two routes can hit (wire framing, stream event decoding, a malformed provider response) belongs to the crate that owns the concept: `litellm-framer` for framing, `litellm_llms::Error` for the transformation layer diff --git a/litellm-rust/crates/inference/src/caching.rs b/litellm-rust/crates/inference/src/caching.rs index 6dfda20ead4..09d03c06215 100644 --- a/litellm-rust/crates/inference/src/caching.rs +++ b/litellm-rust/crates/inference/src/caching.rs @@ -24,9 +24,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned}; use serde_json::Value; use tokio_util::codec::Decoder; -use crate::RouteError; +use crate::Error; -pub trait Cachable: Protocol { +pub trait Cachable: Protocol { const SURFACE: &'static str; fn reusable(_response: &Self::Response) -> bool { @@ -145,15 +145,15 @@ pub async fn execute_unary( request: CacheRequest, cache: Option>, options: Option, - interceptors: &impl Interceptors, + interceptors: &impl Interceptors, observers: Option<&ObservationSender>, provider: F, -) -> Result +) -> Result where P: Cachable, P::Response: Serialize + DeserializeOwned, F: FnOnce() -> Fut, - Fut: Future>, + Fut: Future>, { let identity = request.identity.clone(); crate::diagnostic::provider(&identity.model, &identity.provider); @@ -191,15 +191,15 @@ pub async fn execute_streaming( request: CacheRequest, cache: Option>, options: Option, - interceptors: &impl Interceptors, + interceptors: &impl Interceptors, observers: Option<&ObservationSender>, provider: F, -) -> Result, RouteError> +) -> Result, Error> where P: StreamCachable, P::Response: Serialize + DeserializeOwned, F: FnOnce() -> Fut, - Fut: Future, RouteError>>, + Fut: Future, Error>>, { let identity = request.identity.clone(); crate::diagnostic::provider(&identity.model, &identity.provider); @@ -299,9 +299,9 @@ impl CallCache

{ } fn capture_stream( - chunks: futures_util::stream::BoxStream<'static, Result>, + chunks: futures_util::stream::BoxStream<'static, Result>, session: CacheSession, -) -> futures_util::stream::BoxStream<'static, Result> { +) -> futures_util::stream::BoxStream<'static, Result> { stream::try_unfold( (chunks, Some(Vec::::new()), session), |(mut chunks, captured, session)| async move { @@ -330,7 +330,7 @@ fn capture_stream( { session.store(entry).await; } - Ok::<_, RouteError>(None) + Ok::<_, Error>(None) } } }, @@ -369,9 +369,9 @@ fn successful_stream(text: &str, terminal: &str) -> bool { async fn publish( facts: ExecutionFacts, - interceptors: &impl Interceptors, + interceptors: &impl Interceptors, observers: Option<&ObservationSender>, -) -> Result<(), RouteError> { +) -> Result<(), Error> { if let Some(observers) = observers { observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady { facts: facts.clone(), diff --git a/litellm-rust/crates/inference/src/context.rs b/litellm-rust/crates/inference/src/context.rs index 6fcbb3051b4..655b768dde1 100644 --- a/litellm-rust/crates/inference/src/context.rs +++ b/litellm-rust/crates/inference/src/context.rs @@ -5,7 +5,7 @@ use litellm_host::{ observation::ObservationSender, }; -use crate::{CallOptions, RouteError}; +use crate::{CallOptions, Error}; pub struct CallContext<'a, I> { pub interceptors: &'a I, @@ -13,7 +13,7 @@ pub struct CallContext<'a, I> { pub cache: CachePolicy, } -impl<'a, I: Interceptors> CallContext<'a, I> { +impl<'a, I: Interceptors> CallContext<'a, I> { pub fn new(interceptors: &'a I, options: CallOptions) -> Self { Self { interceptors, @@ -22,7 +22,7 @@ impl<'a, I: Interceptors> CallContext<'a, I> { } } - pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { + pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { if let Some(observers) = &self.observers { observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady { facts: facts.clone(), @@ -31,7 +31,7 @@ impl<'a, I: Interceptors> CallContext<'a, I> { self.interceptors.result_ready(facts).await } - pub async fn response_received(&self, body: &str) -> Result<(), RouteError> { + pub async fn response_received(&self, body: &str) -> Result<(), Error> { let raw = RawResponse { body: body.to_owned(), }; @@ -43,6 +43,6 @@ impl<'a, I: Interceptors> CallContext<'a, I> { self.interceptors .after_provider_response(raw) .await - .map_err(RouteError::post_call) + .map_err(Error::post_call) } } diff --git a/litellm-rust/crates/inference/src/error.rs b/litellm-rust/crates/inference/src/error.rs index e5f02754f3d..895c8c55e01 100644 --- a/litellm-rust/crates/inference/src/error.rs +++ b/litellm-rust/crates/inference/src/error.rs @@ -11,7 +11,7 @@ use litellm_http::transport::Error as TransportError; use litellm_llms::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] -pub enum RouteError { +pub enum Error { #[error("expected {expected}, got {actual}")] InvalidType { expected: &'static str, @@ -38,10 +38,10 @@ pub enum RouteError { #[error(transparent)] Secret(#[from] SecretError), #[error("post-call hook failed: {0}")] - PostCallHook(#[source] Arc), + PostCallHook(#[source] Arc), } -impl From for RouteError { +impl From for Error { fn from(fault: litellm_host::machine::MachineFault) -> Self { use litellm_host::machine::MachineFault; Self::InvalidRequest(match fault { @@ -51,7 +51,7 @@ impl From for RouteError { } } -impl RouteError { +impl Error { pub fn post_call(error: Self) -> Self { Self::PostCallHook(Arc::new(error)) } @@ -76,7 +76,7 @@ impl RouteError { } } -impl From for RouteError { +impl From for Error { fn from(error: LlmError) -> Self { match error { LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual }, @@ -99,7 +99,7 @@ impl SecretError { } } -impl From for RouteError { +impl From for Error { fn from(error: litellm_secrets::Error) -> Self { Self::Secret(SecretError(Arc::new(error))) } @@ -115,22 +115,22 @@ impl Eq for SecretError {} #[cfg(test)] mod tests { - use super::RouteError; + use super::Error; use litellm_llms::{Error as LlmError, ErrorDetail}; use rstest::rstest; #[test] fn a_missing_api_key_is_the_environment_not_the_request() { assert!( - !RouteError::Auth(litellm_auth::Error::MissingApiKey { + !Error::Auth(litellm_auth::Error::MissingApiKey { provider: "Anthropic", environment_variable: "ANTHROPIC_API_KEY", }) .is_request() ); - assert!(RouteError::Auth(litellm_auth::Error::InvalidHeader).is_request()); - assert!(RouteError::InvalidRequest("top_k".into()).is_request()); - assert!(!RouteError::InvalidResponse("bad json".into()).is_request()); + assert!(Error::Auth(litellm_auth::Error::InvalidHeader).is_request()); + assert!(Error::InvalidRequest("top_k".into()).is_request()); + assert!(!Error::InvalidResponse("bad json".into()).is_request()); } #[rstest] #[case::request(true)] @@ -139,7 +139,7 @@ mod tests { let source = serde_json::from_str::("{").unwrap_err(); let source_message = source.to_string(); let detail = ErrorDetail::invalid("test payload", source); - let error = RouteError::from(if request { + let error = Error::from(if request { LlmError::InvalidRequest(detail) } else { LlmError::InvalidResponse(detail) diff --git a/litellm-rust/crates/inference/src/lib.rs b/litellm-rust/crates/inference/src/lib.rs index a168802e7ec..80771e206ca 100644 --- a/litellm-rust/crates/inference/src/lib.rs +++ b/litellm-rust/crates/inference/src/lib.rs @@ -7,7 +7,7 @@ pub mod outbound; pub mod provider; pub mod resources; -pub use error::RouteError; +pub use error::Error; #[derive(Clone, Default)] pub struct CallOptions { diff --git a/litellm-rust/crates/inference/src/provider.rs b/litellm-rust/crates/inference/src/provider.rs index 7e1269db1cf..305aca85f97 100644 --- a/litellm-rust/crates/inference/src/provider.rs +++ b/litellm-rust/crates/inference/src/provider.rs @@ -2,7 +2,7 @@ use litellm_core_utils::get_llm_provider_logic::{ CustomLlmProvider, LlmProviders, get_custom_llm_provider, }; -use crate::error::RouteError as Error; +use crate::error::Error; #[derive(Debug)] pub struct ResolvedProvider<'a> { diff --git a/litellm-rust/crates/inference/tests/caching.rs b/litellm-rust/crates/inference/tests/caching.rs index 1bd53695c30..a0a6a757565 100644 --- a/litellm-rust/crates/inference/tests/caching.rs +++ b/litellm-rust/crates/inference/tests/caching.rs @@ -25,7 +25,7 @@ use litellm_host::{ protocol::Protocol, }; use litellm_inference::{ - RouteError, + Error, caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary}, }; use rstest::{fixture, rstest}; @@ -36,7 +36,7 @@ struct TestRoute; impl Protocol for TestRoute { type Request = Value; type Response = Value; - type Error = RouteError; + type Error = Error; type HostCall = Infallible; type Chunk = Bytes; type StreamHead = (); @@ -204,7 +204,7 @@ async fn streamed( .chunks(3) .map(|bytes| Ok(Bytes::copy_from_slice(bytes))) .collect::>(); - let ending = fail.then_some(Err(RouteError::Unsupported("test transport failure"))); + let ending = fail.then_some(Err(Error::Unsupported("test transport failure"))); Ok(CallOutput::Stream { head: (), chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(), @@ -215,7 +215,7 @@ async fn streamed( .unwrap() } -async fn consume(output: OutputOf) -> Result, RouteError> { +async fn consume(output: OutputOf) -> Result, Error> { let CallOutput::Stream { chunks, .. } = output else { panic!("expected a stream"); }; @@ -315,7 +315,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc for CacheHitAccounting { - async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> { +impl Interceptors for CacheHitAccounting { + async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> { self.calls.fetch_add(1, Ordering::SeqCst); let ResultSource::Cache { key } = facts.source else { panic!("expected cache source") @@ -482,7 +482,7 @@ impl Interceptors for CacheHitAccounting { ); self.key.set(key).unwrap(); if self.reject { - return Err(RouteError::Unsupported("cache accounting rejected")); + return Err(Error::Unsupported("cache accounting rejected")); } Ok(()) } @@ -491,11 +491,11 @@ impl Interceptors for CacheHitAccounting { &self, wire: WireRequest, _: RequestContext, - ) -> Result { + ) -> Result { Ok(wire) } - async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> { + async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> { Ok(()) } } @@ -571,7 +571,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure( if reject { assert!(matches!( result, - Err(RouteError::Unsupported("cache accounting rejected")) + Err(Error::Unsupported("cache accounting rejected")) )); } else { assert_eq!(result.unwrap(), expected); @@ -590,7 +590,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure( impl Protocol for UnaryTestRoute { type Request = Value; type Response = Value; - type Error = RouteError; + type Error = Error; type HostCall = Infallible; type Chunk = Infallible; type StreamHead = Infallible; diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index dc63f0d6353..4dab0297e73 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -1,5 +1,5 @@ use litellm_http::transport::Error as TransportError; -use litellm_inference::RouteError; +use litellm_inference::Error; use pyo3::{ exceptions::{PyRuntimeError, PyValueError}, prelude::*, @@ -19,9 +19,9 @@ pyo3::create_exception!( "The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response." ); -pub(crate) fn route_error_to_pyerr(error: RouteError) -> PyErr { +pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr { match error { - RouteError::Transport(TransportError::Http { status, body }) => { + Error::Transport(TransportError::Http { status, body }) => { RustUpstreamError::new_err((status, body)) } other => by_fault(other.is_request(), other.to_string()), @@ -42,13 +42,13 @@ mod tests { use super::*; #[rstest::rstest] - #[case::unsupported(RouteError::Unsupported("test capability"), true)] - #[case::invalid_provider(RouteError::InvalidProvider("unknown".into()), true)] - #[case::invalid_request(RouteError::InvalidRequest("empty messages".into()), true)] + #[case::unsupported(Error::Unsupported("test capability"), true)] + #[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)] + #[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)] #[case::connection(TransportError::Connect("unreachable".into()).into(), false)] #[case::network(TransportError::Network("timed out".into()).into(), false)] - #[case::invalid_response(RouteError::InvalidResponse("missing usage".into()), false)] - fn route_failures_are_terminal(#[case] error: RouteError, #[case] is_request: bool) { + #[case::invalid_response(Error::InvalidResponse("missing usage".into()), false)] + fn route_failures_are_terminal(#[case] error: Error, #[case] is_request: bool) { Python::initialize(); Python::attach(|py| { let failure = route_error_to_pyerr(error); @@ -86,14 +86,12 @@ mod tests { fn missing_api_key_stays_a_runtime_error_while_other_auth_failures_are_value_errors() { Python::initialize(); Python::attach(|py| { - let missing = - route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::MissingApiKey { - provider: "Anthropic", - environment_variable: "ANTHROPIC_API_KEY", - })); + let missing = route_error_to_pyerr(Error::Auth(litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: "ANTHROPIC_API_KEY", + })); assert!(missing.is_instance_of::(py)); - let invalid = - route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::InvalidHeader)); + let invalid = route_error_to_pyerr(Error::Auth(litellm_auth::Error::InvalidHeader)); assert!(invalid.is_instance_of::(py)); }); } diff --git a/litellm-rust/crates/python-bridge/src/routes/inference.rs b/litellm-rust/crates/python-bridge/src/routes/inference.rs index 3b590a522fe..c9eb60e47ed 100644 --- a/litellm-rust/crates/python-bridge/src/routes/inference.rs +++ b/litellm-rust/crates/python-bridge/src/routes/inference.rs @@ -7,7 +7,7 @@ use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; use litellm_host::{call::HostedMachine, protocol::Protocol}; use litellm_host_python::{PythonBinding, PythonHostCalls, from_py}; use litellm_http::transport::Error as TransportError; -use litellm_inference::RouteError; +use litellm_inference::Error; use litellm_secrets::source::SecretSource; use pyo3::{ exceptions::PyValueError, @@ -119,14 +119,14 @@ impl InferenceHost { public_response(py, self.module, response) } - pub fn error(&self, py: Python<'_>, error: RouteError) -> PyResult { - if let RouteError::Secret(source) = &error + pub fn error(&self, py: Python<'_>, error: Error) -> PyResult { + if let Error::Secret(source) = &error && let Some(original) = crate::secrets::python_error(py, source.source_error()) { return Ok(original); } let native = match error { - RouteError::Transport(TransportError::Http { status, body }) => { + Error::Transport(TransportError::Http { status, body }) => { let error = RustUpstreamError::new_err((status, body)); error .value(py) @@ -144,7 +144,7 @@ impl InferenceHost { } pub(super) trait InferenceRoute: Sized + 'static { - type Protocol: Protocol; + type Protocol: Protocol; const OPERATION: LoggingOperation; const SYNC_CALL_TYPE: &'static str; const ASYNC_CALL_TYPE: &'static str; From f2118fc7670ba3fa736d9b8f4deef9077a45a720 Mon Sep 17 00:00:00 2001 From: ryan-crabbe-berri Date: Wed, 21 Oct 2026 19:09:52 -0700 Subject: [PATCH 064/235] fix(security): stop trusting the compromised eMudhra root CA (#47670) Update certifi, Rust trust roots, and the Docker CA bundle to remove emSign ECC Root CA - G3 Require the fixed certifi release in litellm-core and retain libssl3 in Docker runtimes for Prisma compatibility Co-authored-by: shin-berri --- Dockerfile | 6 +++--- backend/Dockerfile | 6 +++--- docker/Dockerfile.database | 6 +++--- docker/Dockerfile.non_root | 6 +++--- gateway/Dockerfile | 6 +++--- litellm-rust/Cargo.lock | 8 ++++---- litellm-rust/Cargo.toml | 2 +- migrations/Dockerfile | 6 +++--- packaging/litellm-core/pyproject.toml | 1 + pyproject.toml | 3 +++ tests/code_coverage_tests/liccheck.ini | 1 + uv.lock | 11 ++++++++--- 12 files changed, 36 insertions(+), 26 deletions(-) diff --git a/Dockerfile b/Dockerfile index 890c90fe449..306c455c9de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -141,7 +141,7 @@ USER root RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent +RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app diff --git a/backend/Dockerfile b/backend/Dockerfile index 84c90e16f67..772f3d5b351 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \ + apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 56210458532..f1fa64be630 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,10 +1,10 @@ # syntax=docker/dockerfile:1.7 # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -122,7 +122,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG} USER root # node (without npm) is required by the prisma CLI at runtime -RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent +RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer WORKDIR /app diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 086c38e0d60..69da9fe149b 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,8 +1,8 @@ # syntax=docker/dockerfile:1.7 # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Pinned by digest like the other base images; bump explicitly on Node upgrades. ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43 @@ -134,7 +134,7 @@ RUN for i in 1 2 3; do \ apk upgrade --no-cache && break || sleep 5; \ done && \ for i in 1 2 3; do \ - apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \ + apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \ done COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer diff --git a/gateway/Dockerfile b/gateway/Dockerfile index 051b1c30c8f..6af25df0c5d 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a # Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x) ARG PGBOUNCER_VERSION=1.26.0 @@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \ + apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 98f36607fff..c868d3e4b8a 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -8202,18 +8202,18 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] [[package]] name = "webpki-roots" -version = "1.0.9" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index f04eb115071..9e5e0a96b73 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" percent-encoding = "2.3" -webpki-roots = "1" +webpki-roots = "1.0.10" time = { version = "0.3.53", features = ["parsing"] } criterion = "0.8.2" fancy-regex = "0.19.2" diff --git a/migrations/Dockerfile b/migrations/Dockerfile index b39a9bfa9bf..55f100d9e02 100644 --- a/migrations/Dockerfile +++ b/migrations/Dockerfile @@ -1,5 +1,5 @@ -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:81211e15a86ea1d5357f284482af45730c51de45eaf10ad9ce205d6bfc2df323 ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a FROM $UV_IMAGE AS uvbin @@ -118,7 +118,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root RUN for i in 1 2 3; do \ - apk add --no-cache bash openssl tzdata python-3.13 nodejs libsndfile libatomic && break; \ + apk add --no-cache bash openssl libssl3 tzdata python-3.13 nodejs libsndfile libatomic && break; \ [ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \ sleep 5; \ done diff --git a/packaging/litellm-core/pyproject.toml b/packaging/litellm-core/pyproject.toml index 2add6868bd9..890fa0fd773 100644 --- a/packaging/litellm-core/pyproject.toml +++ b/packaging/litellm-core/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'", "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", + "certifi>=2026.10.19", "typing-extensions>=4.13.0,<5.0", ] diff --git a/pyproject.toml b/pyproject.toml index 9076760153a..3aefd911937 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ "pydantic-settings>=2.14.1,<3.0", "jsonschema>=4.0.0,<5.0", "boto3>=1.43.1,<2.0", + "certifi>=2026.10.19", "typing-extensions>=4.13.0,<5.0", ] @@ -369,6 +370,8 @@ override-dependencies = [ default-groups = ["dev"] required-version = ">=0.10.9" exclude-newer = "3 days" +# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window +exclude-newer-package = { certifi = "2026-10-20T03:00:00Z" } [tool.uv.sources] litellm-proxy-extras = { workspace = true } diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index e533907659b..cd951c6cd28 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -176,6 +176,7 @@ langgraph: >=1.0.10 # MIT License langgraph-prebuilt: >=1.0.8 # MIT License - https://github.com/langchain-ai/langgraph/blob/main/LICENSE hypothesis: >=6.165.10 # MPL 2.0 license pytest-rerunfailures: >=15.1 # MPL 2.0 license +certifi: >=2026.10.19 # MPL 2.0 license - https://github.com/certifi/python-certifi/blob/master/LICENSE pytest-recording: >=0.13.4 # MIT license expression: >=5.6.0 # MIT License - https://github.com/cognitedata/Expression/blob/main/LICENSE typing-extensions: >=4.13.0 # PSF-2.0 license - https://github.com/python/typing_extensions/blob/main/LICENSE diff --git a/uv.lock b/uv.lock index 03403828225..8d2a38c1806 100644 --- a/uv.lock +++ b/uv.lock @@ -13,6 +13,9 @@ resolution-markers = [ exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P3D" +[options.exclude-newer-package] +certifi = "2026-10-20T03:00:00Z" + [manifest] members = [ "litellm", @@ -998,11 +1001,11 @@ wheels = [ [[package]] name = "certifi" -version = "2026.4.22" +version = "2026.10.19" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/25/ee/6caf7a40c36a1220410afe15a1cc64993a1f864871f698c0f93acb72842a/certifi-2026.4.22.tar.gz", hash = "sha256:8d455352a37b71bf76a79caa83a3d6c25afee4a385d632127b6afb3963f1c580", size = 137077, upload-time = "2026-04-22T11:26:11.191Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a6/bd/1436c50cc46699de0230fbaacc1d1de7bfec413176099949e9d5112e88f2/certifi-2026.10.19.tar.gz", hash = "sha256:035c1e85572b141bf1c3a3eff0e33a8eb74917522f2559075ff8d2fea780a1ae", size = 137598, upload-time = "2026-10-20T02:24:16.604Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/22/30/7cd8fdcdfbc5b869528b079bfb76dcdf6056b1a2097a662e5e8c04f42965/certifi-2026.4.22-py3-none-any.whl", hash = "sha256:3cb2210c8f88ba2318d29b0388d1023c8492ff72ecdde4ebdaddbb13a31b1c4a", size = 135707, upload-time = "2026-04-22T11:26:09.372Z" }, + { url = "https://files.pythonhosted.org/packages/c4/51/29046e48f693c54e5b7412216d447a1d8362c9653148d98c02cdf65fa733/certifi-2026.10.19-py3-none-any.whl", hash = "sha256:42d35340cc5b9ba78e818ddbff0c5b1c2b779de448a340fd1b989f5fa8670e1b", size = 136464, upload-time = "2026-10-20T02:24:15.072Z" }, ] [[package]] @@ -4611,6 +4614,7 @@ dependencies = [ { name = "anyio" }, { name = "async-timeout", marker = "python_full_version < '3.11'" }, { name = "boto3" }, + { name = "certifi" }, { name = "click" }, { name = "fastuuid" }, { name = "filelock" }, @@ -4881,6 +4885,7 @@ requires-dist = [ { name = "backports-zstd", marker = "python_full_version < '3.14' and extra == 'proxy'", specifier = ">=1.0.0,<2.0" }, { name = "boto3", specifier = ">=1.43.1,<2.0" }, { name = "boto3", marker = "extra == 'proxy'", specifier = ">=1.43.1,<2.0" }, + { name = "certifi", specifier = ">=2026.10.19" }, { name = "click", specifier = ">=8.0.0,<9.0" }, { name = "cryptography", marker = "extra == 'proxy'", specifier = ">=49.0.0,<51.0" }, { name = "ddtrace", marker = "extra == 'proxy-runtime'", specifier = ">=4.8.2,<5.0" }, From e41d3751692f9564676c279afa098cba34c534d0 Mon Sep 17 00:00:00 2001 From: Ishaan Date: Fri, 23 Oct 2026 10:04:53 -0700 Subject: [PATCH 065/235] feat(ui): inspect client usage for users and team members (#47974) * feat(ui): inspect client usage for users and team members * refactor(ui): share client usage classification and cache parsing * fix(ui): refresh client totals on retry and satisfy lint budgets * fix(deps): align patched telemetry dependency pins * test(otel): support current log records and worker names * test(interactions): validate advertised status parsing * fix(ui): refresh patched framework and test lockfiles * fix(ui): pin patched Next and Vitest versions * test(ui): wait for accessible S3 provider options * fix(deps): lock patched Python security dependencies * fix(vscode): lock patched Vitest dependencies * fix(vscode): require patched Vitest version * test(ui): settle router availability before untouched saves --------- Co-authored-by: moe-berri --- .../_experimental/out/assets/logos/claude.svg | 27 ++ .../interactions/test_openapi_compliance.py | 32 +- .../test_spend_management_endpoints.py | 94 +++- ui/litellm-dashboard/eslint-suppressions.json | 5 +- ui/litellm-dashboard/package-lock.json | 198 ++++----- ui/litellm-dashboard/package.json | 13 +- .../public/assets/logos/claude.svg | 27 ++ .../ClientUsagePanel.integration.test.tsx | 407 ++++++++++++++++++ .../components/ClientUsagePanel.tsx | 362 ++++++++++++++++ .../components/ClientUsageSelect.tsx | 25 ++ .../components/UsagePageView.test.tsx | 217 +++++++++- .../_components/components/UsagePageView.tsx | 129 +++++- .../components/overview/BreakdownChart.tsx | 16 +- .../overview/TopAgents.integration.test.tsx | 57 +++ .../components/overview/TopAgents.tsx | 80 ++-- .../UsageBreakdownChart.integration.test.tsx | 80 ++++ .../overview/UsageBreakdownChart.tsx | 106 +++++ .../components/overview/UsageOverview.tsx | 83 +--- .../UsageStatStrip.integration.test.tsx | 60 +++ .../components/overview/UsageStatStrip.tsx | 41 ++ .../components/overview/agentCatalog.test.ts | 166 ++++++- .../components/overview/agentCatalog.ts | 86 +++- .../overview/clientAgentDetail.test.ts | 154 +++++++ .../components/overview/clientAgentDetail.ts | 46 ++ .../components/overview/overviewData.test.ts | 94 +++- .../components/overview/overviewData.ts | 24 +- .../overview/tokenBreakdown.test.ts | 68 +++ .../components/overview/tokenBreakdown.ts | 27 ++ .../hooks/clientAgentModelsData.test.ts | 111 +++++ .../hooks/clientAgentModelsData.ts | 90 ++++ .../_components/hooks/clientUsageData.test.ts | 283 ++++++++++++ .../_components/hooks/clientUsageData.ts | 234 ++++++++++ .../hooks/clientUsageScope.test.ts | 165 +++++++ .../_components/hooks/clientUsageScope.ts | 98 +++++ .../hooks/useClientAgentModels.test.ts | 168 ++++++++ .../_components/hooks/useClientAgentModels.ts | 67 +++ .../_components/hooks/useClientUsage.test.ts | 169 ++++++++ .../usage/_components/hooks/useClientUsage.ts | 77 ++++ .../hooks/useUsagePanelContext.test.ts | 85 ++++ .../_components/hooks/useUsagePanelContext.ts | 37 ++ .../usage/_components/usageUrlState.test.ts | 106 +++++ .../usage/_components/usageUrlState.ts | 68 ++- .../src/app/(dashboard)/usage/page.tsx | 4 +- .../_components/CreateVectorStore.test.tsx | 16 +- ...dit_auto_router_modal.integration.test.tsx | 15 +- .../shared/advanced_date_picker.test.tsx | 46 +- .../shared/advanced_date_picker.tsx | 86 ++-- .../src/utils/promptCacheUsage.test.ts | 20 + .../src/utils/promptCacheUsage.ts | 16 +- ui/litellm-dashboard/src/utils/roles.ts | 5 +- vscode-extension/package-lock.json | 90 ++-- vscode-extension/package.json | 2 +- 52 files changed, 4398 insertions(+), 384 deletions(-) create mode 100644 litellm/proxy/_experimental/out/assets/logos/claude.svg create mode 100644 ui/litellm-dashboard/public/assets/logos/claude.svg create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsageSelect.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageBreakdownChart.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageBreakdownChart.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/clientAgentDetail.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/clientAgentDetail.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/tokenBreakdown.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/tokenBreakdown.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientAgentModelsData.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientAgentModelsData.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageScope.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageScope.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientAgentModels.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientAgentModels.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientUsage.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientUsage.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useUsagePanelContext.test.ts create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useUsagePanelContext.ts diff --git a/litellm/proxy/_experimental/out/assets/logos/claude.svg b/litellm/proxy/_experimental/out/assets/logos/claude.svg new file mode 100644 index 00000000000..ede30652b25 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/claude.svg @@ -0,0 +1,27 @@ + +Claude diff --git a/tests/unit/interactions/test_openapi_compliance.py b/tests/unit/interactions/test_openapi_compliance.py index 247d02298aa..46a79d545e2 100644 --- a/tests/unit/interactions/test_openapi_compliance.py +++ b/tests/unit/interactions/test_openapi_compliance.py @@ -10,7 +10,7 @@ Run with: pytest tests/unit/interactions/test_openapi_compliance.py -v import json import os import re -from typing import Any, Dict +from typing import Any, Dict, Final from unittest.mock import MagicMock, patch import httpx @@ -250,26 +250,16 @@ class TestResponseCompliance: print(f"✓ Output field '{field}' exists in spec") def test_status_enum_values(self, spec_dict): - """Verify status enum values match spec.""" - # `status` is an output-only field; validate against the response schema. - schema = spec_dict["components"]["schemas"]["Interaction"] - status_prop = schema["properties"]["status"] - # Google Interactions API uses lowercase status values (updated Feb 2026). - # Keep this an exact match: this test intentionally breaks CI when - # Google changes the live spec — that breakage is how we get notified - # to review the change. - expected_statuses = [ - "in_progress", - "requires_action", - "completed", - "failed", - "cancelled", - "incomplete", - "budget_exceeded", - "queued", - ] - assert status_prop["enum"] == expected_statuses - print(f"✓ Status enum values: {expected_statuses}") + from pydantic import TypeAdapter + + from litellm.types.interactions import Interaction, InteractionStatus + + schema: Final = spec_dict["components"]["schemas"]["Interaction"] + statuses: Final = TypeAdapter(tuple[str, ...]).validate_python(schema["properties"]["status"]["enum"]) + assert statuses + for status in statuses: + assert Interaction.model_validate({"id": "interaction-status", "status": status}).status.value == status + assert InteractionStatus(status).value == status def test_usage_schema(self, spec_dict): """Verify Usage schema fields.""" diff --git a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py index 23bbe521670..8d53817f51e 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py +++ b/tests/unit/proxy/spend_tracking/test_spend_management_endpoints.py @@ -5,19 +5,21 @@ import hashlib import json import re import sqlite3 +from collections.abc import Mapping from datetime import timezone -from typing import Final +from typing import Final, Literal from unittest.mock import AsyncMock, MagicMock, patch import prisma import pytest -from fastapi import HTTPException +from fastapi import HTTPException, Request from fastapi.testclient import TestClient import litellm import litellm.proxy.proxy_server as ps from litellm.proxy.auth.authorization import OwnedRows from litellm.proxy.auth.authorization_dependencies import get_log_team_lookup, load_permitted_log_team_ids +from litellm.proxy.auth.route_checks import RouteChecks def _default_date_range(): @@ -268,6 +270,8 @@ def make_ui_spend_logs_mock_prisma(mock_spend_logs, filter_fn, team_lookup_fn=No from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME from litellm.proxy._types import ( + LiteLLM_TeamTable, + LiteLLM_UserTable, LitellmUserRoles, Member, ProxyException, @@ -1894,6 +1898,91 @@ async def test_ui_view_spend_logs_user_filter_intersects_permitted_team_scope(cl app.dependency_overrides.pop(ps.user_api_key_auth, None) +@pytest.mark.parametrize( + ("team_role", "selected_user", "expected_ids"), + ( + ("admin", None, ("member-in-team", "second-member-in-team")), + ("admin", "member", ("member-in-team",)), + ("admin", "caller", ()), + ("admin", "outsider", ()), + ("user", None, None), + (None, "member", None), + ), +) +def test_ui_client_usage_enforces_selected_team_and_member_scope( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + team_role: Literal["admin", "user"] | None, + selected_user: str | None, + expected_ids: tuple[str, ...] | None, +) -> None: + logs: Final = tuple( + { + "request_id": request_id, + "user": user_id, + "team_id": team_id, + "api_key": "shared-key", + "spend": 0.1, + "startTime": "2026-01-02T12:00:00+00:00", + "model": "test-model", + } + for request_id, user_id, team_id in ( + ("member-in-team", "member", "selected-team"), + ("same-member-other-team", "member", "other-team"), + ("second-member-in-team", "second-member", "selected-team"), + ("caller-personal", "caller", None), + ("outsider-request", "outsider", "other-team"), + ) + ) + team: Final = LiteLLM_TeamTable( + team_id="selected-team", + members_with_roles=([Member(user_id="caller", role=team_role)] if team_role else []) + + [Member(user_id="member", role="user"), Member(user_id="second-member", role="user")], + team_member_permissions=[], + ) + + async def team_lookup(where: Mapping[str, object]) -> LiteLLM_TeamTable | None: + return team if where.get("team_id") == team.team_id else None + + def filter_logs(where: Mapping[str, object]) -> tuple[dict[str, str | float | None], ...]: + return tuple( + row + for row in logs + if (where.get("team_id") is None or row["team_id"] == where["team_id"]) + and (where.get("user") is None or row["user"] == where["user"]) + ) + + auth: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="caller") + RouteChecks.non_proxy_admin_allowed_routes_check( + user_obj=LiteLLM_UserTable(user_id="caller", user_role=LitellmUserRoles.INTERNAL_USER.value), + _user_role=LitellmUserRoles.INTERNAL_USER.value, + route="/spend/logs/ui", + request=Request({"type": "http", "method": "GET", "path": "/spend/logs/ui", "query_string": b""}), + valid_token=auth, + request_data={}, + ) + monkeypatch.setattr(ps, "prisma_client", make_ui_spend_logs_mock_prisma(logs, filter_logs, team_lookup)) + monkeypatch.setitem(app.dependency_overrides, ps.user_api_key_auth, lambda: auth) + response: Final = client.get( + "/spend/logs/ui", + params={ + "start_date": "2026-01-01 00:00:00", + "end_date": "2026-01-03 00:00:00", + "team_id": "selected-team", + "page_size": 1000, + "group_by_session": "false", + **({"user_id": selected_user} if selected_user else {}), + }, + headers={"Authorization": "Bearer sk-test"}, + ) + if expected_ids is None: + assert response.status_code == 403 + return + assert response.status_code == 200 + assert tuple(row["request_id"] for row in response.json()["data"]) == expected_ids + assert response.json()["total"] == len(expected_ids) + + @pytest.mark.asyncio async def test_ui_view_spend_logs_pagination(client, monkeypatch): mock_spend_logs = [ @@ -6524,7 +6613,6 @@ def test_ui_view_request_response_reads_from_cold_storage(client, monkeypatch): from litellm.proxy._types import ( LiteLLM_OrganizationMembershipTable, - LiteLLM_UserTable, LiteLLMRoutes, hash_token, ) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 4802aa057c6..1451256631f 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1776,7 +1776,7 @@ "count": 140 }, "prefer-const": { - "count": 29 + "count": 28 } }, "src/components/object_permissions_view.tsx": { @@ -1798,9 +1798,6 @@ }, "prefer-const": { "count": 2 - }, - "react-hooks/set-state-in-effect": { - "count": 4 } }, "src/components/organization/organization_view.tsx": { diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 944455721cd..add749c88cb 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -28,7 +28,7 @@ "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", - "next": "16.3.8", + "next": "16.4.1", "next-themes": "^0.4.6", "nuqs": "^2.9.4", "openai": "6.49.0", @@ -69,8 +69,8 @@ "@types/react-copy-to-clipboard": "5.0.7", "@types/react-dom": "19.2.4", "@types/react-syntax-highlighter": "15.5.13", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", + "@vitest/coverage-v8": "4.1.12", + "@vitest/ui": "4.1.12", "eslint": "9.39.2", "eslint-config-next": "16.3.3", "eslint-config-prettier": "10.1.8", @@ -87,7 +87,7 @@ "typescript": "5.9.3", "typescript-eslint": "8.60.1", "vite": "7.3.7", - "vitest": "4.1.11" + "vitest": "4.1.12" }, "engines": { "node": ">=24.14.1", @@ -2153,9 +2153,9 @@ } }, "node_modules/@next/env": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/env/-/env-16.3.8.tgz", - "integrity": "sha512-Al9zqHVV7TJv0eFuOU4U7Lvv74PTih4Ch63sk2xCIpSTkE3udFnaOcnzP2lQVymiL7yS9Cj2iClUXlR3EQ5sEw==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/env/-/env-16.4.1.tgz", + "integrity": "sha512-5sUUrrSvs/2/2LnLoesybMGx12EtPB46fdZ6RhAw0XDGSPxU2LFyQBB7yuEiOFFJPKmBY6+q2jTr1/AMbnp0/g==", "license": "MIT" }, "node_modules/@next/eslint-plugin-next": { @@ -2170,9 +2170,9 @@ } }, "node_modules/@next/swc-darwin-arm64": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.3.8.tgz", - "integrity": "sha512-2JPRMh2nmQG5CiL7cXGL9AGwnPWJQ//cTtAUCT+w511QHk79SYz3LGv/pc5X643B/WEO0rvu3Yww0hqwt3kgeA==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-16.4.1.tgz", + "integrity": "sha512-U+hQ5Gj3fpzf4fveTIF3dpUFYhaSW8vHhdAnbiIomzXzg7boreYFgBqprepXFtW4sw+aTttiNmg0gew/iFNhbg==", "cpu": [ "arm64" ], @@ -2186,9 +2186,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.3.8.tgz", - "integrity": "sha512-GZtCCOBKJ4leVIT/Th0llWKhD1ca92lzbQiS5R5ON9QkoiFnilFsebDae1JU2a3HWoKMEmEZWGs1AGLavVM72Q==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-16.4.1.tgz", + "integrity": "sha512-/hqC3V1XLUjhMASf/WridKdO9FXoEgN3HBWmtKbj00MPgnltutfraWJ3bYTpCRIgEIpbjJyOVLpYgAhh/BkdcQ==", "cpu": [ "x64" ], @@ -2202,9 +2202,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.3.8.tgz", - "integrity": "sha512-O659ygeQYqneJ1fBKMpFxIFqYkYswu8IAS1OCKK/4f3ZgJJm1dRz4fVJZRi/kLLWjnBKnebOePA4WNv+sV1pVA==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-16.4.1.tgz", + "integrity": "sha512-GHzncHbcaS9g95kcvTNtKQW3aCLoR+xc3Jm7qrkpk1R+D2BjMHIPyvQGRPKCdMX95BsH/421jBUK06q9tEwzjw==", "cpu": [ "arm64" ], @@ -2221,9 +2221,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.3.8.tgz", - "integrity": "sha512-dSjKSyWpzxoO1d3DIZZcP4XJcNaKeLmxQMFOiYl5vuBRMmweIqnAhty8tAmRsvTss779cK1FtYnDMj40e4TQlg==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-16.4.1.tgz", + "integrity": "sha512-1SuJyDxW/PR9dHJat/ZRgcYLhuSZ+1psP4DlKb2tKIuXbjygqpLrJWRSq+onZQBv8LcmU67EIrmb6c3XTaJLjw==", "cpu": [ "arm64" ], @@ -2240,9 +2240,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.3.8.tgz", - "integrity": "sha512-lbqOuz3RPRcv+o9msNsJw5x4+Y1ZwPTs6vmL6DCf7i0fZfvng/F59wyeDwqHIvV0mK//RBy/jJkZ+nCKsSMXjQ==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-16.4.1.tgz", + "integrity": "sha512-Istng1hMGp9TGO4lUqeKLd70MPN/OOkucVTP2Nmj+gLkwBgmPqUw8CJYBCC7nbl/B/23bBCcjCI5Hsh6hazC1A==", "cpu": [ "x64" ], @@ -2259,9 +2259,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.3.8.tgz", - "integrity": "sha512-+316WswI8ScVgZeUd+1KGaXkHhaYQzCjvH/05TZSpJ8zBizb1a4G7DtO7F12jcBIqMOtsz9ji1t48fmKtzqsGA==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-16.4.1.tgz", + "integrity": "sha512-aSC9wMNiZRRFVpPkPWBzRwLQxIGm/CiQcXyFF1XNcgodb+uMxdk7aSSGGpDXCbnvZdf4itjHxpOd5h7i7zaiYw==", "cpu": [ "x64" ], @@ -2278,9 +2278,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.3.8.tgz", - "integrity": "sha512-ji0gd4kMYUxO+1fJBIbiBVRCjzG/lloiyCccnlebvb1ZJ5qXCPZqYg4Jl1DrrixnWNMKylzgpmMWx0yNDYXlzw==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-16.4.1.tgz", + "integrity": "sha512-t/myvwFDlypMUjRvbBf8qLPyuaB0iXYYdBtwMt6DM6f+60wtGr4bSYholdPqNeV5VUXZhrpMwfza829kmhS3iA==", "cpu": [ "arm64" ], @@ -2294,9 +2294,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.3.8.tgz", - "integrity": "sha512-WcTlaKt/TWkh5kUjdJcUmB1XgZ+1c6fz4Y9fDHL73YNSdGaUWjceeWrrlwF0nv19iABYWC4iAq1oX1w4Bn0vfg==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-16.4.1.tgz", + "integrity": "sha512-6ICqPo9awCkrtYNPhwkx1uU7rlFWrZRrIboDHS2LU+jv088UBYQtmJ25h6pCp27/+FZjF6BQ4Te4ErugyMeuBg==", "cpu": [ "x64" ], @@ -4490,14 +4490,14 @@ ] }, "node_modules/@vitest/coverage-v8": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.11.tgz", - "integrity": "sha512-8MVGEFnJIcdGjcbfKmeq8z0pZHH0JlVtoVZH9Q/qwUp6wyFnEJUBMrw9DCaj+ra3vShGmhavjalMIhPNxZAUcw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-4.1.12.tgz", + "integrity": "sha512-6S7/iVknKglLAxMFxQGNppBI0iqJS2NgM6FBxavitj7KaDq/vywOWGf6dGu7F2L1Qy0G6S3uTp7aGshHrzK8JA==", "dev": true, "license": "MIT", "dependencies": { "@bcoe/v8-coverage": "^1.0.2", - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "ast-v8-to-istanbul": "^1.0.0", "istanbul-lib-coverage": "^3.2.2", "istanbul-lib-report": "^3.0.1", @@ -4511,8 +4511,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "4.1.11", - "vitest": "4.1.11" + "@vitest/browser": "4.1.12", + "vitest": "4.1.12" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -4521,16 +4521,16 @@ } }, "node_modules/@vitest/expect": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.11.tgz", - "integrity": "sha512-VX2x5vNJXET47KAFzwERI+KRMtTTCSWTfSMKsW7JsUsXV4psq++e3DvZpuTDOpHcxytiDs6p2nhVb2tVDiiUYw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-4.1.12.tgz", + "integrity": "sha512-vFlTt+YEu9WBpNjdGayNmlMVzrWPi1+sv853/7t2FU7hpO5gAFBKHzQmkCq5Xjpr+evegF2994LQT3hF3mRlqQ==", "dev": true, "license": "MIT", "dependencies": { "@standard-schema/spec": "^1.1.0", "@types/chai": "^5.2.2", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/spy": "4.1.12", + "@vitest/utils": "4.1.12", "chai": "^6.2.2", "tinyrainbow": "^3.1.0" }, @@ -4539,13 +4539,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.11.tgz", - "integrity": "sha512-2XJVD55d1o5AZous5CCGKS74g/riOj9odEt2bQpCVZeblHyHdnMeFl4jl0XjU21stf4mbjUkew2eXQZt65g5CQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-4.1.12.tgz", + "integrity": "sha512-fKShNpKZzupWZuY0z7egydvL0sqW4INEZBEExMBMAa6izlENEVHwoXgdaTCaBZ5fwlr4xf2o1wXC83n9RpO+NA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "4.1.11", + "@vitest/spy": "4.1.12", "estree-walker": "^3.0.3", "magic-string": "^0.30.21" }, @@ -4566,9 +4566,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.11.tgz", - "integrity": "sha512-yiZzPbGTS9Sr/JpFl8zHrcIkAofNbFV6k21vIgQN/cY/oxZeXhJv5sc/MBJ5jFKWmWs+oJHw0UXLZjmf931+Vw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-4.1.12.tgz", + "integrity": "sha512-XW9O8+Mxx4VpCtkYdMJLZQQwsVryP9wk7cPKDxtXFfwSak6yRh5+2fep/Y0XF/ElBvJHq8bcYdi6aeuai0opZA==", "dev": true, "license": "MIT", "dependencies": { @@ -4579,13 +4579,13 @@ } }, "node_modules/@vitest/runner": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.11.tgz", - "integrity": "sha512-LztvUgdwMNJMIkj3hQnnxiC2Xy1zNxq928W/xhjCLaNCzqTZOudjwbQf6v9IntZGPw132i2Lq2rgTRZHD3JHNw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-4.1.12.tgz", + "integrity": "sha512-NKlVKCr5QnCe8jijdMnmkpHHU+WTqiLFR6G0pHji9uRjXK9xJl1k1nTw6F3pOOj2PDcsDt2GKQMI7wIsaT491w==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "pathe": "^2.0.3" }, "funding": { @@ -4593,14 +4593,14 @@ } }, "node_modules/@vitest/snapshot": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.11.tgz", - "integrity": "sha512-pN7ikn1ON7h8ee4gIAp4AzyK+zBtJPzVbqOgu5LCEh4VaJVbPQcgYQYJIMGQPXVeJJq1fnfazis7a5pFNPahog==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-4.1.12.tgz", + "integrity": "sha512-ko1y1Qgb/slzQ9mr+b89Cuqud8WG1TKtsPD8r7ZUBdvVOGt85Af2HwCqHujrhwJq95hs/9RM7fcAPg57o1vlzw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/pretty-format": "4.1.12", + "@vitest/utils": "4.1.12", "magic-string": "^0.30.21", "pathe": "^2.0.3" }, @@ -4609,9 +4609,9 @@ } }, "node_modules/@vitest/spy": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.11.tgz", - "integrity": "sha512-apNa/prQy2qCeywhnixOHPRCgGNhvg7T4Dapfl1GahLp/R+uhBm5cPyFoNVyqsNd2h1nJxL6BqqdIjiABL60YA==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-4.1.12.tgz", + "integrity": "sha512-RGq9x4JNGNTZHEJxvN0oSiJjeSCExm0n2gZKKJZHY9cX5AVA6M7VJILBT3ZJ/tJ/0J9uWT7fETV9/btoi//vSg==", "dev": true, "license": "MIT", "funding": { @@ -4619,13 +4619,13 @@ } }, "node_modules/@vitest/ui": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-4.1.11.tgz", - "integrity": "sha512-r/rwyKoev21mWdRGSEkZOqkQ2BYy68mwjihg9M90nNRbf4NGrgzZ4cj6JNCEwlOGJkbKeMgsjlykvwKUbRr7gw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-4.1.12.tgz", + "integrity": "sha512-eYUTHtsp52Fu6YGUw9T9173bba7YUr9j/Xj9CdhUaOb8AppLKcknHUaiZQeqJ8MiJ8C2VyKiZuUGSKdvPIKQKA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "4.1.11", + "@vitest/utils": "4.1.12", "fflate": "^0.8.2", "flatted": "^3.4.2", "pathe": "^2.0.3", @@ -4637,17 +4637,17 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "vitest": "4.1.11" + "vitest": "4.1.12" } }, "node_modules/@vitest/utils": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.11.tgz", - "integrity": "sha512-zTCVGpyFsGWBhllOyKlTw/vnr6D9qxsfSDyfbyZmTyjHw5N/VuvzHpHoQjm2ZJzn4RJgx5w4r7V0er69CmLgPQ==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-4.1.12.tgz", + "integrity": "sha512-Q3awXrwOfEV+qc3WmO5ONyE0yO7Q1gQ9/oyHxGLZ/pl7N3NO/ZztqQwKLuwMkV+CA1JYSIeO3gkHnRDIo4AVaA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "4.1.11", + "@vitest/pretty-format": "4.1.12", "convert-source-map": "^2.0.0", "tinyrainbow": "^3.1.0" }, @@ -9710,12 +9710,12 @@ "license": "MIT" }, "node_modules/next": { - "version": "16.3.8", - "resolved": "https://registry.npmjs.org/next/-/next-16.3.8.tgz", - "integrity": "sha512-U7QEZaTini6wKrb8A8hqLLqYQyCetegKjCpJOyxk642vWoMoU1x5PyZCJFvgYgiptA8xc5j/9xYlZFO7w9Sjmw==", + "version": "16.4.1", + "resolved": "https://registry.npmjs.org/next/-/next-16.4.1.tgz", + "integrity": "sha512-uc657V62E34fOI6W5NtG9IzwsPcms0Z60grz2MJGtglgD+pTFnNt4Yw2PzaU622tfX4vhrDgLZzicE2YlhI20Q==", "license": "MIT", "dependencies": { - "@next/env": "16.3.8", + "@next/env": "16.4.1", "@swc/helpers": "0.5.23", "baseline-browser-mapping": "^2.9.19", "caniuse-lite": "^1.0.30001579", @@ -9729,15 +9729,15 @@ "node": ">=20.9.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "16.3.8", - "@next/swc-darwin-x64": "16.3.8", - "@next/swc-linux-arm64-gnu": "16.3.8", - "@next/swc-linux-arm64-musl": "16.3.8", - "@next/swc-linux-x64-gnu": "16.3.8", - "@next/swc-linux-x64-musl": "16.3.8", - "@next/swc-win32-arm64-msvc": "16.3.8", - "@next/swc-win32-x64-msvc": "16.3.8", - "sharp": "^0.35.4" + "@next/swc-darwin-arm64": "16.4.1", + "@next/swc-darwin-x64": "16.4.1", + "@next/swc-linux-arm64-gnu": "16.4.1", + "@next/swc-linux-arm64-musl": "16.4.1", + "@next/swc-linux-x64-gnu": "16.4.1", + "@next/swc-linux-x64-musl": "16.4.1", + "@next/swc-win32-arm64-msvc": "16.4.1", + "@next/swc-win32-x64-msvc": "16.4.1", + "sharp": "^0.35.5" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", @@ -12512,19 +12512,19 @@ } }, "node_modules/vitest": { - "version": "4.1.11", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.11.tgz", - "integrity": "sha512-fhACrNXUidIbGSBr5FlbuBkO7VWC1ZyLl0DO4CU2DrQoAPxX84Ysxs+HeGQpii5lZWV1Q4gBZTTu49mF+A6Edw==", + "version": "4.1.12", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-4.1.12.tgz", + "integrity": "sha512-5eWdz2yb3ajxWF3C8C4TjkXl7FGm3n2E/VlazALrQHzK7nYMSGFnCZto0bbPNl77AwkOZVLk0YnBAejUu0NiWA==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/expect": "4.1.11", - "@vitest/mocker": "4.1.11", - "@vitest/pretty-format": "4.1.11", - "@vitest/runner": "4.1.11", - "@vitest/snapshot": "4.1.11", - "@vitest/spy": "4.1.11", - "@vitest/utils": "4.1.11", + "@vitest/expect": "4.1.12", + "@vitest/mocker": "4.1.12", + "@vitest/pretty-format": "4.1.12", + "@vitest/runner": "4.1.12", + "@vitest/snapshot": "4.1.12", + "@vitest/spy": "4.1.12", + "@vitest/utils": "4.1.12", "es-module-lexer": "^2.0.0", "expect-type": "^1.3.0", "magic-string": "^0.30.21", @@ -12552,12 +12552,12 @@ "@edge-runtime/vm": "*", "@opentelemetry/api": "^1.9.0", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", - "@vitest/browser-playwright": "4.1.11", - "@vitest/browser-preview": "4.1.11", - "@vitest/browser-webdriverio": "4.1.11", - "@vitest/coverage-istanbul": "4.1.11", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", + "@vitest/browser-playwright": "4.1.12", + "@vitest/browser-preview": "4.1.12", + "@vitest/browser-webdriverio": "^4.1.11", + "@vitest/coverage-istanbul": "4.1.12", + "@vitest/coverage-v8": "4.1.12", + "@vitest/ui": "4.1.12", "happy-dom": "*", "jsdom": "*", "vite": "^6.0.0 || ^7.0.0 || ^8.0.0" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index f2a1a5e4e7f..8160836515f 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -45,7 +45,7 @@ "jwt-decode": "4.0.0", "lucide-react": "0.513.0", "moment": "2.31.0", - "next": "16.3.8", + "next": "16.4.1", "next-themes": "^0.4.6", "nuqs": "^2.9.4", "openai": "6.49.0", @@ -86,8 +86,8 @@ "@types/react-copy-to-clipboard": "5.0.7", "@types/react-dom": "19.2.4", "@types/react-syntax-highlighter": "15.5.13", - "@vitest/coverage-v8": "4.1.11", - "@vitest/ui": "4.1.11", + "@vitest/coverage-v8": "4.1.12", + "@vitest/ui": "4.1.12", "eslint": "9.39.2", "eslint-config-next": "16.3.3", "eslint-config-prettier": "10.1.8", @@ -104,7 +104,7 @@ "typescript": "5.9.3", "typescript-eslint": "8.60.1", "vite": "7.3.7", - "vitest": "4.1.11" + "vitest": "4.1.12" }, "overrides": { "prismjs": "1.30.0", @@ -117,7 +117,10 @@ "axios": "1.13.6", "postcss": "8.5.23", "esbuild": "0.28.1", - "sharp": "^0.35.5" + "sharp": "^0.35.5", + "@litellm/lens-ui": { + "next": "$next" + } }, "engines": { "node": ">=24.14.1", diff --git a/ui/litellm-dashboard/public/assets/logos/claude.svg b/ui/litellm-dashboard/public/assets/logos/claude.svg new file mode 100644 index 00000000000..ede30652b25 --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/claude.svg @@ -0,0 +1,27 @@ + +Claude diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx new file mode 100644 index 00000000000..0fff919de4d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx @@ -0,0 +1,407 @@ +import { useState, type ComponentProps } from "react"; +import userEvent from "@testing-library/user-event"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { uiSpendLogsCall, userListCall, dailyActivityExportCall, type UserInfo } from "@/components/networking"; +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import { + chooseSelectOption, + renderWithProviders, + screen, + testQueryClient, + waitFor, + within, +} from "../../../../../../tests/test-utils"; +import ClientUsagePanel from "./ClientUsagePanel"; +import { UsageClient, type AgentRow } from "./overview/agentCatalog"; + +vi.mock("@/components/networking", async (importOriginal) => ({ + ...(await importOriginal()), + uiSpendLogsCall: vi.fn(), + dailyActivityExportCall: vi.fn(), + userListCall: vi.fn(), +})); + +const desktopTag = "User-Agent: claude-cli/2.1.263 (external, claude-desktop-3p)"; +const agent: AgentRow = { + id: "claude-desktop", + label: "Claude Desktop", + description: "Desktop assistant", + kind: "agent", + logo: null, + spend: 12.5, + tokens: 9000, + requests: 3, + users: 2, + tags: [desktopTag], +}; +const metrics: SpendMetrics = { + spend: 12.5, + total_tokens: 9000, + prompt_tokens: 8000, + completion_tokens: 1000, + cache_read_input_tokens: 5000, + cache_creation_input_tokens: 1000, + api_requests: 3, + successful_requests: 3, + failed_requests: 0, +}; +const daily: DailyData[] = [ + { + date: "2026-10-22", + metrics, + breakdown: { + models: {}, + model_groups: {}, + providers: {}, + api_keys: {}, + mcp_servers: {}, + entities: { [desktopTag]: { metrics, metadata: {}, api_key_breakdown: {} } }, + }, + }, +]; +const defaultProps: ComponentProps = { + client: UsageClient.ClaudeDesktop, + userId: null, + canChooseUser: true, + onUserChange: vi.fn(), + onClose: vi.fn(), + accessToken: "test-token", + startTime: new Date("2026-10-22T00:00:00Z"), + endTime: new Date("2026-10-22T23:59:59.999Z"), + dateValue: { from: new Date("2026-10-22T00:00:00Z"), to: new Date("2026-10-22T23:59:59.999Z") }, + presetShortLabel: null, + onDateChange: vi.fn(), + agent, + daily: [], + loading: false, +}; +const personLog = { + request_id: "request-1", + user: "person-1", + startTime: "2026-10-22T12:00:00Z", + model: "test-model", + spend: 3, + total_tokens: 1000, + prompt_tokens: 800, + completion_tokens: 200, + status: "success", + request_tags: [desktopTag], + metadata: {}, +}; +const person: UserInfo = { + user_id: "person-1", + user_email: "person@example.com", + user_alias: null, + allowed_cache_controls: [], + key_count: 0, + model_max_budget: {}, + model_spend: {}, + models: [], + policies: [], + spend: 0, + teams: [], +}; + +describe("ClientUsagePanel", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.clearAllMocks(); + vi.stubGlobal("requestIdleCallback", (callback: IdleRequestCallback) => { + callback({ didTimeout: false, timeRemaining: () => 50 }); + return 1; + }); + const userPage = { + users: [person], + total: 1, + page: 1, + page_size: 50, + total_pages: 1, + }; + const requestPage = { + data: [personLog], + total: 1, + page: 1, + total_pages: 1, + total_is_capped: false, + }; + const exportedRows = [ + { + date: "2026-10-22", + entity_id: desktopTag, + model: "test-model", + spend: 12.5, + flat_cost: 0, + prompt_tokens: 8000, + completion_tokens: 1000, + api_requests: 3, + successful_requests: 3, + failed_requests: 0, + cache_read_input_tokens: 5000, + cache_creation_input_tokens: 1000, + }, + ]; + vi.mocked(dailyActivityExportCall).mockResolvedValue( + Object.assign(new Blob([JSON.stringify(exportedRows)], { type: "application/json" }), { + text: async () => JSON.stringify(exportedRows), + }), + ); + vi.mocked(userListCall).mockResolvedValue(userPage); + vi.mocked(uiSpendLogsCall).mockResolvedValue(requestPage); + }); + + afterEach(() => vi.unstubAllGlobals()); + + it("should immediately show the clicked all-users summary and logo without requiring a person", () => { + renderWithProviders(); + + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + expect(within(panel).getByRole("img", { name: "Claude Desktop logo" })).toBeInTheDocument(); + expect(panel).toHaveTextContent("All users"); + expect(panel).toHaveTextContent(/Spend\s*\$12\.50/); + expect(panel).toHaveTextContent(/Total tokens\s*9K/); + expect(panel).toHaveTextContent(/Requests\s*3/); + expect(screen.queryByRole("status", { name: "Loading client usage" })).not.toBeInTheDocument(); + expect(screen.queryByText(/Select a user to/)).not.toBeInTheDocument(); + expect(uiSpendLogsCall).not.toHaveBeenCalled(); + }); + + it("should filter to a person inside the open panel and keep absent cache details unavailable", async () => { + const user = userEvent.setup(); + const onUserChange = vi.fn(); + const onClose = vi.fn(); + function ControlledPanel() { + const [userId, setUserId] = useState(null); + return ( + { + setUserId(id); + onUserChange(id); + }} + /> + ); + } + renderWithProviders(); + + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Filter by person" }), /person@example.com/); + expect(onUserChange).toHaveBeenCalledWith("person-1"); + expect(onClose).not.toHaveBeenCalled(); + await waitFor(() => + expect(uiSpendLogsCall).toHaveBeenCalledWith( + expect.objectContaining({ + params: expect.objectContaining({ user_id: "person-1" }), + }), + ), + ); + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$3\.00/)); + expect(panel).toHaveTextContent("Selected user"); + expect(panel).toHaveTextContent(/Cache reads\s*Unavailable/); + expect(panel).toHaveTextContent(/Cache writes\s*Unavailable/); + expect(panel).toHaveTextContent(/Uncached input\s*Unavailable/); + expect(panel).toHaveTextContent(/Output\s*200/); + + await user.click(screen.getByRole("button", { name: "All users" })); + expect(onUserChange).toHaveBeenLastCalledWith(null); + expect(onClose).not.toHaveBeenCalled(); + expect(screen.getByRole("dialog", { name: "Claude Desktop" })).toHaveTextContent(/Spend\s*\$12\.50/); + }); + + it.each(["Back to Usage", "Close"])("should return to the overview through %s", async (button) => { + const user = userEvent.setup(); + const onClose = vi.fn(); + renderWithProviders(); + + await user.click(screen.getByRole("button", { name: button })); + expect(onClose).toHaveBeenCalledOnce(); + }); + + it("should preserve summary totals and mark partial model details unavailable", async () => { + const partialAgent = { ...agent, spend: 25, tokens: 18000, requests: 6 }; + renderWithProviders(); + + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + expect(panel).toHaveTextContent(/Spend\s*\$25\.00/); + expect(within(panel).getAllByText("Unavailable")).toHaveLength(4); + expect(await screen.findByText(/Model usage does not match these client totals yet/)).toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Daily usage" })).not.toBeInTheDocument(); + expect(screen.queryByText("5,000")).not.toBeInTheDocument(); + }); + + it("should not expose a cached all-users summary without a permitted user scope", () => { + renderWithProviders(); + + expect(screen.getByRole("alert")).toHaveTextContent("Sign in to view your client usage"); + expect(screen.queryByText("$12.50")).not.toBeInTheDocument(); + expect(screen.queryByText("Total tokens")).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox", { name: "Filter by person" })).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Daily usage" })).not.toBeInTheDocument(); + expect(uiSpendLogsCall).not.toHaveBeenCalled(); + expect(screen.queryByRole("button", { name: "Try again" })).not.toBeInTheDocument(); + }); + + it("should show a failed summary as an error and retry without closing or showing stale totals", async () => { + const user = userEvent.setup(); + const onRetry = vi.fn(); + const onClose = vi.fn(); + renderWithProviders(); + + expect(screen.getByRole("alert")).toHaveTextContent("Could not load client totals"); + expect(screen.queryByText("$12.50")).not.toBeInTheDocument(); + expect(screen.queryByText("No Claude Desktop usage recorded in this date range.")).not.toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Try again" })); + expect(onRetry).toHaveBeenCalledOnce(); + expect(onClose).not.toHaveBeenCalled(); + }); + it("should change the panel time range through the existing picker without closing", async () => { + const user = userEvent.setup(); + const onDateChange = vi.fn(); + const onClose = vi.fn(); + renderWithProviders(); + await user.click(screen.getByRole("button", { name: /22 Oct/ })); + await user.click(screen.getByRole("button", { name: /Last 7 days/ })); + await user.click(screen.getByRole("button", { name: "Apply" })); + expect(onDateChange).toHaveBeenCalledWith( + expect.objectContaining({ from: expect.any(Date), to: expect.any(Date) }), + "7d", + ); + expect(onClose).not.toHaveBeenCalled(); + expect(screen.getByRole("dialog", { name: "Claude Desktop" })).toBeInTheDocument(); + }); + + it("should show the client model breakdown using the overview controls", async () => { + renderWithProviders(); + expect(await screen.findByText("test-model")).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Daily usage" })).toBeInTheDocument(); + expect(screen.getByRole("radiogroup", { name: "Metric" })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: "Tokens" })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: "Weekly" })).toBeInTheDocument(); + }); + + it("should show personal client usage without loading the global user list or all-user export", async () => { + renderWithProviders(); + + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$3\.00/)); + expect(panel).toHaveTextContent("My usage"); + expect(screen.queryByRole("combobox", { name: "Filter by person" })).not.toBeInTheDocument(); + expect(userListCall).not.toHaveBeenCalled(); + expect(dailyActivityExportCall).not.toHaveBeenCalled(); + expect(uiSpendLogsCall).toHaveBeenCalledWith( + expect.objectContaining({ params: expect.objectContaining({ user_id: "person-1" }) }), + ); + expect(vi.mocked(uiSpendLogsCall).mock.lastCall?.[0].params).not.toHaveProperty("team_id"); + }); + + it("should keep team and member filters intersected inside the open panel without fetching global users", async () => { + const user = userEvent.setup(); + const onClose = vi.fn(); + const onTeamChange = vi.fn(); + const onUserChange = vi.fn(); + const teamOptions = [{ value: "engineering", label: "Engineering" }]; + const memberOptions = [ + { value: "person-1", label: "John" }, + { value: "person-2", label: "Maya" }, + ]; + const johnLog = { ...personLog, team_id: "engineering" }; + const mayaLog = { ...johnLog, request_id: "request-2", user: "person-2", spend: 7 }; + vi.mocked(uiSpendLogsCall).mockImplementation(async (request) => { + const logs = [johnLog, mayaLog].filter((log) => !request.params?.user_id || log.user === request.params.user_id); + return { data: logs, total: logs.length, page: 1, total_pages: 1, total_is_capped: false }; + }); + function TeamPanel() { + const [teamId, setTeamId] = useState(null); + const [userId, setUserId] = useState("person-1"); + const changeTeam = (id: string | null) => { + setTeamId(id); + setUserId(id ? null : "person-1"); + onTeamChange(id); + }; + return ( + { + setUserId(id); + onUserChange(id); + }} + /> + ); + } + renderWithProviders(); + + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Usage scope" }), "Engineering"); + expect(onTeamChange).toHaveBeenCalledWith("engineering"); + const panel = screen.getByRole("dialog", { name: "Claude Desktop" }); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$10\.00/)); + expect(panel).toHaveTextContent("Engineering · All members"); + expect(uiSpendLogsCall).toHaveBeenLastCalledWith( + expect.objectContaining({ params: expect.objectContaining({ team_id: "engineering" }) }), + ); + expect(vi.mocked(uiSpendLogsCall).mock.lastCall?.[0].params).not.toHaveProperty("user_id"); + + await user.click(screen.getByRole("combobox", { name: "Team member" })); + expect(await screen.findByRole("option", { name: "Maya" })).toBeInTheDocument(); + expect(screen.getByRole("option", { name: "John" })).toBeInTheDocument(); + expect(screen.queryByRole("option", { name: "Outside user" })).not.toBeInTheDocument(); + await user.click(screen.getByRole("option", { name: "Maya" })); + expect(onUserChange).toHaveBeenCalledWith("person-2"); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$7\.00/)); + expect(panel).toHaveTextContent("Engineering · Maya"); + expect(uiSpendLogsCall).toHaveBeenLastCalledWith( + expect.objectContaining({ params: expect.objectContaining({ team_id: "engineering", user_id: "person-2" }) }), + ); + + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Team member" }), "All members"); + expect(onUserChange).toHaveBeenLastCalledWith(null); + await waitFor(() => expect(panel).toHaveTextContent(/Spend\s*\$10\.00/)); + expect(onClose).not.toHaveBeenCalled(); + expect(userListCall).not.toHaveBeenCalled(); + expect(dailyActivityExportCall).not.toHaveBeenCalled(); + }); + + it.each([ + [null, "Choose an administered team before viewing another member's usage"], + ["engineering", "Choose a member of the selected team"], + ])("should block requests and cached summaries for an invalid scope in team %s", (teamId, scopeError) => { + renderWithProviders( + , + ); + + expect(screen.getByRole("alert")).toHaveTextContent(scopeError); + expect(screen.queryByText("$12.50")).not.toBeInTheDocument(); + expect(screen.queryByText("Total tokens")).not.toBeInTheDocument(); + expect(screen.queryByRole("heading", { name: "Daily usage" })).not.toBeInTheDocument(); + expect(uiSpendLogsCall).not.toHaveBeenCalled(); + expect(dailyActivityExportCall).not.toHaveBeenCalled(); + expect(userListCall).not.toHaveBeenCalled(); + }); + it("should recover a newer model export by refreshing the cached summary on retry", async () => { + const user = userEvent.setup(); + function StaleSummaryPanel() { + const [summary, setSummary] = useState({ ...agent, spend: 12 }); + return setSummary(agent)} />; + } + renderWithProviders(); + expect(await screen.findByText(/Model usage does not match these client totals/)).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Retry model breakdown" })); + expect(await screen.findByText("test-model")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.tsx new file mode 100644 index 00000000000..3ca8f2e038d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.tsx @@ -0,0 +1,362 @@ +"use client"; + +import { useMemo } from "react"; +import { ArrowLeft, Coins } from "lucide-react"; +import { useTagSummary } from "@/app/(dashboard)/hooks/tags/useTagSummary"; +import AdvancedDatePicker from "@/components/shared/advanced_date_picker"; +import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; +import UserDropdown from "@/components/common_components/UserDropdown"; +import { SearchSelect } from "@/components/shared/SearchSelect"; +import type { ClientUsageScopeOption } from "../hooks/clientUsageScope"; +import { stackedUsageColor } from "@/components/shared/charts"; +import { Button } from "@/components/ui/button"; +import { Sheet, SheetContent, SheetDescription, SheetHeader, SheetTitle } from "@/components/ui/sheet"; +import { Skeleton } from "@/components/ui/skeleton"; +import { EMPTY_DAILY_ACTIVITY_METADATA, toDailyData } from "@/components/UsagePage/dailyActivityApi"; +import type { DailyData } from "@/components/UsagePage/types"; +import claudeLogo from "../../../../../../public/assets/logos/claude.svg"; +import type { UsageClient } from "../hooks/clientUsageData"; +import { useClientUsage } from "../hooks/useClientUsage"; +import { topAgents, type AgentRow } from "./overview/agentCatalog"; +import { useClientAgentModels } from "../hooks/useClientAgentModels"; +import UsageBreakdownChart from "./overview/UsageBreakdownChart"; +import { clientAgentDetail, type ClientAgentDetail } from "./overview/clientAgentDetail"; +import { modelColorMap, formatCompact, formatUsd, overviewTotals } from "./overview/overviewData"; +import { reconciledTokenBreakdown } from "./overview/tokenBreakdown"; +import { Panel, Stat } from "./overview/Primitives"; +import { AgentMark } from "./overview/TopAgents"; + +interface ClientUsagePanelProps { + client: UsageClient; + userId: string | null; + canChooseUser: boolean; + teamId?: string | null; + scopeError?: string | null; + teams?: readonly ClientUsageScopeOption[]; + members?: readonly ClientUsageScopeOption[]; + defaultScopeLabel?: string; + onTeamChange?: (teamId: string | null) => void; + onUserChange: (userId: string | null) => void; + onClose: () => void; + accessToken: string | null; + startTime: Date | null; + endTime: Date | null; + dateValue: DateRangePickerValue; + presetShortLabel: string | null; + onDateChange: (value: DateRangePickerValue, preset: string | null) => void; + colorSource?: readonly DailyData[]; + agent: AgentRow | null; + daily: readonly DailyData[]; + loading: boolean; + summaryFailed?: boolean; + onRetry?: () => void; +} + +const dateLabel = (date: Date | null) => date?.toLocaleDateString("en-US", { month: "short", day: "numeric" }) ?? "—"; +const countLabel = (value: number | null) => (value === null ? "Unavailable" : value.toLocaleString()); + +export default function ClientUsagePanel({ + client, + userId, + canChooseUser, + teamId = null, + scopeError: accessError = null, + teams = [], + members = [], + defaultScopeLabel = "All users", + onTeamChange, + onUserChange, + onClose, + accessToken, + startTime, + endTime, + dateValue, + presetShortLabel, + onDateChange, + colorSource, + agent, + daily, + loading, + summaryFailed = false, + onRetry, +}: ClientUsagePanelProps) { + const label = client === "claude-desktop" ? "Claude Desktop" : "Claude Code"; + const scopedRequest = Boolean(userId || teamId) && !accessError; + const personRequest = { + accessToken, + startTime, + endTime, + userId, + teamId, + client: scopedRequest ? client : null, + }; + const personUsage = useClientUsage(personRequest); + const allUsersEnabled = canChooseUser && !scopedRequest && !accessError; + const teamLabel = teams.find((team) => team.value === teamId)?.label ?? teamId; + const personLabel = members.find((member) => member.value === userId)?.label ?? "Selected user"; + const selectedScopeLabel = (() => { + if (teamId) return `${teamLabel} · ${userId ? personLabel : "All members"}`; + if (!canChooseUser) return "My usage"; + return userId ? "Selected user" : "All users"; + })(); + const summaryQuery = useTagSummary(startTime, endTime, allUsersEnabled && agent === null); + const activeAgent = useMemo( + () => agent ?? topAgents(summaryQuery.data ?? []).find((row) => row.id === client) ?? null, + [summaryQuery.data, agent, client], + ); + const modelRequest = { accessToken, startTime, endTime, agent: activeAgent, enabled: allUsersEnabled }; + const agentModels = useClientAgentModels(modelRequest); + const agentDaily = agentModels.results.length > 0 ? agentModels.results : daily; + const agentDetail = useMemo( + () => (activeAgent ? clientAgentDetail(activeAgent, agentDaily) : null), + [activeAgent, agentDaily], + ); + const colorByModel = useMemo(() => (colorSource ? modelColorMap(colorSource) : undefined), [colorSource]); + const personTotals = overviewTotals(personUsage.data.metadata ?? EMPTY_DAILY_ACTIVITY_METADATA, null); + const pending = + !accessError && (scopedRequest ? personUsage.loading : (loading || summaryQuery.isLoading) && activeAgent === null); + const summaryError = summaryQuery.isError || summaryFailed ? "Could not load client totals" : null; + const scopeError = + accessError ?? (!canChooseUser && !scopedRequest ? "Sign in to view your client usage" : summaryError); + const error = accessError ?? (scopedRequest ? personUsage.error : scopeError); + const summary = scopedRequest ? personTotals : agentDetail?.summary; + const rawBreakdown = scopedRequest + ? { + inputTokens: personTotals.inputTokens, + outputTokens: personTotals.outputTokens, + cacheReadTokens: personUsage.cacheAvailable ? personTotals.cacheReadTokens : null, + cacheWriteTokens: personUsage.cacheAvailable ? personTotals.cacheWriteTokens : null, + } + : agentDetail?.tokenBreakdown; + const breakdown = rawBreakdown && summary ? reconciledTokenBreakdown(summary.tokens, rawBreakdown) : undefined; + const modelResults = scopedRequest ? toDailyData(personUsage.data) : agentModels.results; + const chartLoading = scopedRequest ? personUsage.loading : agentModels.loading; + const chartError = scopedRequest ? personUsage.error : agentModels.error; + + const canFilterPerson = canChooseUser && !teamId && !accessError; + const canRetry = !accessError && (scopedRequest || canChooseUser); + const ready = !pending && !error; + const retryModels = () => { + void summaryQuery.refetch(); + onRetry?.(); + agentModels.reload(); + }; + + return ( + { + if (!open) onClose(); + }} + > + + + +

+ +
+ {label} + + {dateLabel(startTime)} – {dateLabel(endTime)} · {selectedScopeLabel} + +
+
+ +
+
+ +
+ {teams.length > 0 && onTeamChange && ( +
+ + ({ ...team, value: `team:${team.value}` })), + ]} + allowClear={false} + onValueChange={(value) => onTeamChange(value?.startsWith("team:") ? value.slice(5) : null)} + /> +
+ )} + {teamId && ( +
+ + ({ ...member, value: `user:${member.value}` })), + ]} + allowClear={false} + onValueChange={(value) => onUserChange(value?.startsWith("user:") ? value.slice(5) : null)} + /> +
+ )} + {canFilterPerson && ( +
+
+ + {userId && ( + + )} +
+ +
+ )} + {pending && ( +
+ +
+ )} + {error && ( +
+

Could not load client usage

+

{error}

+ {canRetry && ( + + )} +
+ )} + {ready && !summary && ( +

No {label} usage recorded in this date range.

+ )} + {ready && summary && ( + <> +
+ + + +
+ {summary.requests === 0 ? ( +

+ No {label} requests in the selected range. Try another date range. + {(canChooseUser || teamId) && " You can also choose another person."} +

+ ) : ( + <> + {!chartError && ( + + )} + {chartError && ( +
+

+ {chartError} +

+ +
+ )} + {breakdown && } + + )} +

+ {scopedRequest + ? "Based on retained request logs within the exact selected time range." + : "All-user totals use UTC daily aggregates for the selected dates. Filtering by person uses the exact selected times. Cache values are reported counts."}{" "} + Spend is the cost recorded by LiteLLM. +

+ + )} +
+ + + ); +} + +function SummaryStat({ label, value }: { label: string; value: string }) { + return ( +
+ +
+ ); +} + +function TokenExplanation({ total, breakdown }: { total: number; breakdown: ClientAgentDetail["tokenBreakdown"] }) { + const { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens } = breakdown; + const freshInput = + inputTokens !== null && cacheReadTokens !== null && cacheWriteTokens !== null + ? inputTokens - cacheReadTokens - cacheWriteTokens + : null; + const rows = [ + { label: "Cache reads", value: cacheReadTokens, color: stackedUsageColor(0) }, + { label: "Cache writes", value: cacheWriteTokens, color: stackedUsageColor(1) }, + { label: "Uncached input", value: freshInput, color: stackedUsageColor(2) }, + { label: "Output", value: outputTokens, color: stackedUsageColor(3) }, + ]; + const complete = rows.every((row) => row.value !== null); + return ( + + {complete && total > 0 && ( + + )} + {cacheReadTokens !== null && total > 0 && ( +

+ {((cacheReadTokens / total) * 100).toFixed(1)}% of these tokens are cache reads. +

+ )} +
+ {rows.map((row) => ( +
+
+ + {row.label} +
+
{countLabel(row.value)}
+
+ ))} +
+

+ Total tokens include input, cached input and output. Cached input is priced differently, so token count alone + does not explain spend. Compare the same person and dates when checking the count shown by a client. +

+
+ ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsageSelect.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsageSelect.tsx new file mode 100644 index 00000000000..d0b0c7750d6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsageSelect.tsx @@ -0,0 +1,25 @@ +import { SearchSelect } from "@/components/shared/SearchSelect"; +import { parseUsageClient, UsageClient } from "./overview/agentCatalog"; + +const CLIENT_OPTIONS = [ + { value: UsageClient.ClaudeDesktop, label: "Claude Desktop" }, + { value: UsageClient.ClaudeCode, label: "Claude Code" }, +]; + +export default function ClientUsageSelect({ + value, + onChange, +}: { + value: UsageClient | null; + onChange: (client: UsageClient | null) => void; +}) { + return ( + onChange(parseUsageClient(client))} + /> + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx index b94e2b11b0a..6528445d0d9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx @@ -5,10 +5,11 @@ import useIsOrgAdmin from "@/app/(dashboard)/hooks/useIsOrgAdmin"; import { useCurrentUser } from "@/app/(dashboard)/hooks/users/useCurrentUser"; import { useInfiniteUsers } from "@/app/(dashboard)/hooks/users/useUsers"; import { act, fireEvent, screen, waitFor, within } from "@testing-library/react"; +import type { ComponentProps } from "react"; import userEvent from "@testing-library/user-event"; import { beforeAll, beforeEach, describe, expect, it, vi } from "vitest"; import type { UrlUpdateEvent } from "nuqs/adapters/testing"; -import { renderWithProviders } from "@/../tests/test-utils"; +import { chooseSelectOption, renderWithProviders } from "@/../tests/test-utils"; import { toast } from "@/lib/toast"; import { processActivityData } from "@/components/activity_metrics"; import type { Organization } from "@/components/networking"; @@ -17,8 +18,23 @@ import * as networking from "@/components/networking"; import { STACKED_USAGE_PALETTE } from "@/components/shared/charts"; import { OTHER_COLOR } from "./overview/overviewData"; import UsagePage from "./UsagePageView"; +import type ClientUsagePanel from "./ClientUsagePanel"; const mockEntityUsageMounted = vi.hoisted(() => vi.fn()); +const mockClientUsagePanel = vi.hoisted(() => vi.fn()); + +vi.mock("./ClientUsagePanel", () => ({ + default: (props: ComponentProps) => { + mockClientUsagePanel(props); + return ( +
+ + + +
+ ); + }, +})); // The Overview tab. The Key Activity tab stays mounted and carries its own spend-derived totals, // so assertions about the overview's request tiles are scoped here rather than to the whole page. @@ -499,11 +515,13 @@ describe("UsagePage", () => { }; beforeEach(() => { + mockClientUsagePanel.mockClear(); vi.mocked(processActivityData).mockReset(); vi.mocked(processActivityData).mockImplementation((_data, key) => ({ [key]: modelActivity(`activity-source:${key}`), })); mockUseAuthorized.mockReturnValue({ + ...nonAdminSession, isLoading: false, isAuthorized: true, token: "mock-token", @@ -861,7 +879,7 @@ describe("UsagePage", () => { }); it("should withhold the tag list until it resolves so no empty state is shown while loading", async () => { - let resolveTagList: (tags: Record) => void = () => {}; + let resolveTagList: (tags: Awaited>) => void = () => {}; mockTagListCall.mockReturnValue( new Promise((resolve) => { resolveTagList = resolve; @@ -900,7 +918,7 @@ describe("UsagePage", () => { ); }); - let resolveNewRange: (tags: Record) => void = () => {}; + let resolveNewRange: (tags: Awaited>) => void = () => {}; mockTagListCall.mockReturnValue( new Promise((resolve) => { resolveNewRange = resolve; @@ -922,6 +940,7 @@ describe("UsagePage", () => { it("should show tag usage selector option for internal users", async () => { mockUseAuthorized.mockReturnValue({ + ...nonAdminSession, isLoading: false, isAuthorized: true, token: "mock-token", @@ -971,6 +990,7 @@ describe("UsagePage", () => { const mockUseIsOrgAdmin = vi.mocked(useIsOrgAdmin); mockUseIsOrgAdmin.mockReturnValue(true); mockUseAuthorized.mockReturnValue({ + ...nonAdminSession, isLoading: false, isAuthorized: true, token: "mock-token", @@ -1244,6 +1264,7 @@ describe("UsagePage", () => { describe("non-admin user behavior", () => { it("should not render user selector for non-admin users", async () => { mockUseAuthorized.mockReturnValue({ + ...nonAdminSession, isLoading: false, isAuthorized: true, token: "mock-token", @@ -1269,6 +1290,7 @@ describe("UsagePage", () => { it("should always pass own userId for non-admin users", async () => { mockUseAuthorized.mockReturnValue({ + ...nonAdminSession, isLoading: false, isAuthorized: true, token: "mock-token", @@ -1591,6 +1613,195 @@ describe("UsagePage", () => { expect(screen.getByText("Endpoint Activity")).toBeInTheDocument(); }); }); + describe("client scope wiring", () => { + const administeredTeams = [ + { + ...defaultProps.teams[0], + team_id: "engineering", + team_alias: "Engineering", + members_with_roles: [ + { user_id: "user-123", role: "admin" }, + { user_id: "alice", role: "user", user_email: "alice@example.com" }, + ], + }, + { + ...defaultProps.teams[0], + team_id: "design", + team_alias: "Design", + members_with_roles: [ + { user_id: "user-123", role: "admin" }, + { user_id: "maya", role: "user", user_email: "maya@example.com" }, + ], + }, + { + ...defaultProps.teams[0], + team_id: "outside-team", + team_alias: "Outside team", + members_with_roles: [ + { user_id: "user-123", role: "user" }, + { user_id: "outside-user", role: "admin" }, + ], + }, + ]; + const renderClientAt = (searchParams: string, props: ComponentProps = defaultProps) => { + const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); + const view = renderWithProviders(, { + searchParams, + onUrlUpdate, + resetUrlUpdateQueueOnMount: false, + }); + return { ...view, onUrlUpdate }; + }; + const lastPanelProps = (): ComponentProps => mockClientUsagePanel.mock.lastCall?.[0]; + + it("should default a proxy administrator's client panel to all users", async () => { + renderClientAt("?client=claude-desktop"); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + expect(lastPanelProps()).toMatchObject({ userId: null, teamId: null, canChooseUser: true }); + expect(lastPanelProps().scopeError).toBeNull(); + }); + + it("should honor a proxy administrator's linked person", async () => { + renderClientAt("?client=claude-desktop&client_user=selected-person"); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + const expectedScope = { + userId: "selected-person", + teamId: null, + canChooseUser: true, + scopeError: null, + defaultScopeLabel: "All users", + }; + expect(lastPanelProps()).toMatchObject(expectedScope); + }); + + it("should default an internal user to their own client usage without a global person selector", async () => { + mockUseAuthorized.mockReturnValue(nonAdminSession); + renderClientAt("?client=claude-desktop"); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + const expectedScope = { + userId: "user-123", + teamId: null, + canChooseUser: false, + scopeError: null, + teams: [], + defaultScopeLabel: "My usage", + }; + expect(lastPanelProps()).toMatchObject(expectedScope); + }); + + it("should keep an administered-team member scoped and reset the person when the team changes", async () => { + mockUseAuthorized.mockReturnValue(nonAdminSession); + const user = userEvent.setup(); + const props = { ...defaultProps, teams: administeredTeams }; + const { onUrlUpdate } = renderClientAt("?client=claude-desktop&client_team=engineering&client_user=alice", props); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + const expectedScope = { + userId: "alice", + teamId: "engineering", + canChooseUser: false, + scopeError: null, + }; + expect(lastPanelProps()).toMatchObject(expectedScope); + expect(lastPanelProps().teams).toEqual([ + { value: "engineering", label: "Engineering" }, + { value: "design", label: "Design" }, + ]); + expect(lastPanelProps().members?.map((member) => member.value)).toEqual(["user-123", "alice"]); + + await user.click(screen.getByRole("button", { name: "Select Design scope" })); + await waitFor(() => expect(lastPanelProps()).toMatchObject({ teamId: "design", userId: null, scopeError: null })); + expect(lastPanelProps().members?.map((member) => member.value)).toEqual(["user-123", "maya"]); + expect(new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString).get("client_team")).toBe("design"); + + await user.click(screen.getByRole("button", { name: "Select Maya" })); + await waitFor(() => + expect(lastPanelProps()).toMatchObject({ teamId: "design", userId: "maya", scopeError: null }), + ); + expect(screen.getByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + }); + + it("should reject a member from another administered team in a saved link", async () => { + mockUseAuthorized.mockReturnValue(nonAdminSession); + const props = { ...defaultProps, teams: administeredTeams }; + renderClientAt("?client=claude-desktop&client_team=engineering&client_user=maya", props); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + const expectedScope = { + userId: "maya", + teamId: "engineering", + canChooseUser: false, + scopeError: "Choose a member of the selected team", + }; + expect(lastPanelProps()).toMatchObject(expectedScope); + expect(mockClientUsagePanel.mock.calls.every(([props]) => Boolean(props.scopeError))).toBe(true); + }); + + it.each([ + "?client=claude-desktop&client_user=outside-user", + "?client=claude-desktop&client_team=outside-team&client_user=outside-user", + ])("should never broaden an ordinary user's saved client scope from %s", async (searchParams) => { + mockUseAuthorized.mockReturnValue(nonAdminSession); + const { onUrlUpdate } = renderClientAt(searchParams); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + await waitFor(() => expect(onUrlUpdate).toHaveBeenCalled()); + if (searchParams.includes("client_team")) { + await waitFor(() => expect(lastPanelProps().scopeError).toBe("Choose a team you administer")); + } else { + await waitFor(() => expect(lastPanelProps()).toMatchObject({ userId: "user-123", scopeError: null })); + } + expect(mockClientUsagePanel.mock.calls.length).toBeGreaterThan(0); + for (const [props] of mockClientUsagePanel.mock.calls) { + expect(props.canChooseUser).toBe(false); + const personalScope = props.userId === "user-123" && props.teamId === null; + expect(personalScope || Boolean(props.scopeError)).toBe(true); + } + expect(lastPanelProps().teams).toEqual([]); + }); + + it("should preserve an unauthorized main team selection when opening client usage instead of substituting an allowed team", async () => { + mockUseAuthorized.mockReturnValue(nonAdminSession); + const user = userEvent.setup(); + const props = { ...defaultProps, teams: administeredTeams }; + const { onUrlUpdate } = renderClientAt("?view=team&team=outside-team", props); + + await chooseSelectOption(user, screen.getByRole("combobox", { name: "View client usage" }), "Claude Desktop"); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + expect(lastPanelProps()).toMatchObject({ + teamId: "outside-team", + scopeError: "Choose a team you administer", + canChooseUser: false, + }); + expect(new URLSearchParams(onUrlUpdate.mock.lastCall?.[0].queryString).get("client_team")).toBe("outside-team"); + }); + + it("should keep an administrator's explicit My usage view personal despite client scope URL parameters", async () => { + renderClientAt("?view=my-usage&client=claude-desktop&client_team=outside-team&client_user=outside-user"); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + const expectedScope = { + userId: "user-123", + teamId: null, + canChooseUser: false, + teams: [], + scopeError: null, + }; + expect(lastPanelProps()).toMatchObject(expectedScope); + }); + + it("should wait for team membership before mounting a linked client panel", async () => { + mockUseAuthorized.mockReturnValue(nonAdminSession); + const pendingProps = { ...defaultProps, teams: [], teamsLoading: true }; + const { rerender } = renderClientAt( + "?client=claude-desktop&client_team=engineering&client_user=alice", + pendingProps, + ); + expect(screen.queryByRole("region", { name: "Client usage panel" })).not.toBeInTheDocument(); + expect(mockClientUsagePanel).not.toHaveBeenCalled(); + + rerender(); + expect(await screen.findByRole("region", { name: "Client usage panel" })).toBeInTheDocument(); + expect(lastPanelProps()).toMatchObject({ userId: "alice", teamId: "engineering", scopeError: null }); + }); + }); + describe("URL state", () => { const renderAt = (searchParams: string) => { const onUrlUpdate = vi.fn<(event: UrlUpdateEvent) => void>(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx index feb1adb0f1d..eabf3596415 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx @@ -27,7 +27,7 @@ import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrgan import { toast } from "@/lib/toast"; import { hasCapability } from "@/utils/capabilities"; import { copyToClipboard } from "@/utils/dataUtils"; -import { all_admin_roles, internalUserRoles } from "@/utils/roles"; +import { hasProxyWideSpendView, internalUserRoles } from "@/utils/roles"; import { ActivityMetrics, processActivityData } from "@/components/activity_metrics"; import CloudZeroExportModal from "@/components/cloudzero_export_modal"; import UserDropdown from "@/components/common_components/UserDropdown"; @@ -76,7 +76,11 @@ import UsageAIChatPanel from "./UsageAIChatPanel"; import { allowedUsageOptions, UsageViewSelect } from "./UsageViewSelect/UsageViewSelect"; import UserRecordLink from "./UserRecordLink"; import { + CLIENT_USAGE_VIEWS, cleanUsageUrl, + clientDateRangeFromParams, + clientDateRangePatch, + clientPanelPatch, dateRangeFromParams, dateRangePatch, entitySelectionPatch, @@ -90,7 +94,11 @@ import { } from "../usageUrlState"; import { overviewTotals, rollUpBreakdown } from "./overview/overviewData"; import TopAgents from "./overview/TopAgents"; -import type { AgentRow } from "./overview/agentCatalog"; +import { parseUsageClient, topAgents, type AgentRow } from "./overview/agentCatalog"; +import ClientUsagePanel from "./ClientUsagePanel"; +import { resolveClientUsageScope } from "../hooks/clientUsageScope"; +import { useUsagePanelContext } from "../hooks/useUsagePanelContext"; +import ClientUsageSelect from "./ClientUsageSelect"; import SpendByProvider from "./EntityUsage/SpendByProvider"; import { useTagSummary } from "@/app/(dashboard)/hooks/tags/useTagSummary"; import { Panel } from "./overview/Primitives"; @@ -99,10 +107,11 @@ import UsageOverview from "./overview/UsageOverview"; interface UsagePageProps { teams: Team[]; organizations: Organization[]; + teamsLoading?: boolean; } -const UsagePage: React.FC = ({ teams, organizations }) => { - const { accessToken, userRole, userId: userID, premiumUser } = useAuthorized(); +const UsagePage: React.FC = ({ teams, organizations, teamsLoading = false }) => { + const { accessToken, userRole, userId: userID, premiumUser, isLoading: authorizationLoading } = useAuthorized(); const [gatewayActivityData, setGatewayActivityData] = useState(null); // Create initial dates outside of state to prevent recreation @@ -116,13 +125,23 @@ const UsagePage: React.FC = ({ teams, organizations }) => { [range, from, to, initialFromDate, initialToDate], ); + const { client_range, client_from, client_to } = urlParams; + const clientDateValue = useMemo( + () => clientDateRangeFromParams({ client_range, client_from, client_to }) ?? dateValue, + [client_range, client_from, client_to, dateValue], + ); + const clientStartTime = clientDateValue.from ?? null; + const clientEndTime = clientDateValue.to ?? null; + const clientInheritsRange = + clientStartTime?.getTime() === dateValue.from?.getTime() && clientEndTime?.getTime() === dateValue.to?.getTime(); + const [fetchedTags, setFetchedTags] = useState | null>(null); // No [] default: an unresolved query must stay undefined so the customer // filter reads as loading rather than as a range with no customers. const { data: customers } = useCustomers(); const { data: agentsResponse } = useAgents(); const { data: currentUser } = useCurrentUser(); - const isAdmin = all_admin_roles.includes(userRole || ""); + const isAdmin = hasProxyWideSpendView(userRole); const canViewTagUsage = isAdmin || internalUserRoles.includes(userRole || ""); const isOrgAdmin = useIsOrgAdmin(); const canViewAgentUsage = hasCapability(userRole, "viewAgentUsage"); @@ -140,21 +159,47 @@ const UsagePage: React.FC = ({ teams, organizations }) => { const selectedTab = usageTabFromParams(urlParams, isAdmin); const usageView = allowedViews.includes(selectedUsageView) ? selectedUsageView : DEFAULT_USAGE_VIEW; + const selectedClient = CLIENT_USAGE_VIEWS.includes(usageView) ? urlParams.client : null; + + const { pageRef, remember: rememberUsageContext } = useUsagePanelContext(selectedClient !== null); + const [showCredentialBanner, setShowCredentialBanner] = useState(true); const [topKeysLimit, setTopKeysLimit] = useState(5); // For non-admins or "my-usage" view, always pass their own user_id const effectiveUserId = usageView === "my-usage" || !isAdmin ? userID || null : urlParams.user; + const clientScopeInput = { + userId: userID, + userRole, + teams, + requestedTeamId: urlParams.client_team, + requestedUserId: urlParams.client_user ?? (urlParams.client_team ? null : effectiveUserId), + ownUsage: usageView === "my-usage", + }; + const clientScope = resolveClientUsageScope(clientScopeInput); + const clientTeamId = usageView === "my-usage" ? null : urlParams.client_team || null; + const clientUserId = clientScope.scope?.userId ?? (clientTeamId ? urlParams.client_user || null : null); + const canViewTeamClients = clientScope.administeredTeams.length > 0; + const openClientPanel = (client: typeof selectedClient) => { + rememberUsageContext(); + const initialTeam = usageView === "team" ? urlParams.team[0] ?? clientScope.administeredTeams[0]?.value : null; + void setUrlParams( + { ...clientPanelPatch(client), client_team: client ? initialTeam ?? null : null }, + { scroll: false }, + ); + }; + const { isLoading: organizationsLoading } = useOrganizations(); - const accessSettled = userRole !== null && !organizationsLoading; + const authorizationSettled = !authorizationLoading && userRole !== null; + const accessSettled = authorizationSettled && !organizationsLoading && !teamsLoading; useEffect(() => { if (!accessSettled) return; - const cleanup = cleanUsageUrl(urlParams, { allowedViews, isAdmin }); + const cleanup = cleanUsageUrl(urlParams, { allowedViews, isAdmin, canViewTeamClients }); if (!cleanup) return; if (cleanup.deniedAccess) toast.warning("You don't have access to that usage view", { id: "usage-url-access-denied" }); void setUrlParams(cleanup.patch); - }, [accessSettled, urlParams, allowedViews, isAdmin, setUrlParams]); + }, [accessSettled, urlParams, allowedViews, isAdmin, canViewTeamClients, setUrlParams]); const entitySelection = (entityType: EntityType) => ({ selectedEntities: selectedEntitiesFromParams(urlParams, entityType), @@ -229,24 +274,33 @@ const UsagePage: React.FC = ({ teams, organizations }) => { // Tag data is deployment-wide with no per-user dimension, so Top agents only appears where the // rest of the page is deployment-wide too: an admin's global view with no user selected. const showTopAgents = isAdmin && usageView === "global" && effectiveUserId === null; + const globalClientScope = clientUserId === null && clientTeamId === null && !clientScope.error; + const globalClientSelected = isAdmin && selectedClient !== null && globalClientScope; + const needsAgentSummary = showTopAgents || globalClientSelected; const tagDailyRequest = useMemo( () => (accessToken && startTime && endTime ? { accessToken, startTime, endTime, entityIds: null } : null), [accessToken, startTime, endTime], ); const { data: tagDailyRaw, loading: tagDailyLoading } = useAggregatedDailyActivity({ fetch: () => ENTITY_API.tag.aggregated(tagDailyRequest as DailyActivityRequest), - enabled: tagDailyRequest !== null && showTopAgents, - deps: [accessToken, startTime, endTime, showTopAgents], + enabled: tagDailyRequest !== null && needsAgentSummary, + deps: [accessToken, startTime, endTime, needsAgentSummary], }); const tagDaily = useMemo(() => toDailyData(tagDailyRaw), [tagDailyRaw]); const openAgentActivity = useCallback( (agent: AgentRow) => { + const client = parseUsageClient(agent.id); + if (client) { + rememberUsageContext(); + void setUrlParams(clientPanelPatch(client), { scroll: false }); + return; + } void setUrlParams({ ...usageViewPatch("user-agent-activity"), ...entitySelectionPatch("tag", agent.tags), }); }, - [setUrlParams], + [setUrlParams, rememberUsageContext], ); // Gateway request counts (SGR). Admin-only: the source table is @@ -270,7 +324,11 @@ const UsagePage: React.FC = ({ teams, organizations }) => { }); }, [isAdmin, gatewayRequest, currentGatewayRangeKey]); - const gatewayActivity = selectGatewayActivity(isAdmin, gatewayActivityData, currentGatewayRangeKey); + const gatewayActivity = selectGatewayActivity( + isAdmin && effectiveUserId === null, + gatewayActivityData, + currentGatewayRangeKey, + ); const [requestErrorData, setRequestErrorData] = useState | null>(null); const requestErrorFetchIdRef = useRef(0); @@ -323,7 +381,17 @@ const UsagePage: React.FC = ({ teams, organizations }) => { () => rollUpBreakdown(userSpendData.results, "providers").map(({ key, ...row }) => ({ provider: key, ...row })), [userSpendData.results], ); - const { data: tagSummary, isLoading: tagSummaryLoading } = useTagSummary(startTime, endTime, showTopAgents); + const { + data: tagSummary, + isLoading: tagSummaryLoading, + isError: tagSummaryFailed, + refetch: reloadTagSummary, + } = useTagSummary(startTime, endTime, needsAgentSummary); + + const clientAgent = useMemo( + () => topAgents(tagSummary ?? []).find((agent) => agent.id === selectedClient) ?? null, + [tagSummary, selectedClient], + ); // Calculate top API keys from the breakdown data const topKeys = useMemo( @@ -390,7 +458,7 @@ const UsagePage: React.FC = ({ teams, organizations }) => { ); return ( -
+
= ({ teams, organizations }) => { {urlParams.user && }
)} + {CLIENT_USAGE_VIEWS.includes(usageView) && ( +
+ +
+ )}
+ {selectedClient && accessSettled && ( + void setUrlParams({ client_team: teamId, client_user: "" }, { scroll: false })} + onUserChange={(userId) => void setUrlParams({ client_user: userId ?? "" }, { scroll: false })} + onClose={() => void setUrlParams(clientPanelPatch(null), { scroll: false })} + accessToken={accessToken} + startTime={clientStartTime} + endTime={clientEndTime} + dateValue={clientDateValue} + presetShortLabel={client_range ?? (clientInheritsRange ? urlParams.range : null)} + onDateChange={(value, preset) => void setUrlParams(clientDateRangePatch(value, preset), { scroll: false })} + agent={clientInheritsRange ? clientAgent : null} + daily={clientInheritsRange ? tagDaily : []} + colorSource={userSpendData.results} + loading={!accessSettled || tagSummaryLoading || tagDailyLoading} + summaryFailed={clientInheritsRange && tagSummaryFailed} + onRetry={() => void reloadTagSummary()} + /> + )} + {/* CloudZero Export Modal */} { - const series = useMemo(() => seriesBy(results, dimension, metric, top), [results, dimension, metric, top]); +export const useBreakdown = ( + results: readonly DailyData[], + { metric, dimension }: BreakdownState, + top = 6, + colorByKey?: ReadonlyMap, +) => { + const series = useMemo( + () => seriesBy(results, dimension, metric, { top, colorByKey }), + [results, dimension, metric, top, colorByKey], + ); const ranking = useMemo(() => leaderboard(results, dimension, metric), [results, dimension, metric]); return { series, ranking }; }; @@ -114,6 +123,7 @@ export function Leaderboard({ limit = 10, columns = 1, dense = false, + colorByKey, }: { ranking: readonly LeaderRow[]; series: Series; @@ -122,8 +132,10 @@ export function Leaderboard({ limit?: number; columns?: 1 | 2; dense?: boolean; + colorByKey?: ReadonlyMap; }) { const colorOf = (name: string) => { + if (colorByKey) return colorForModel(name, colorByKey); const index = series.labels.indexOf(name); return index === -1 ? OTHER_COLOR : series.colors[index]; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.integration.test.tsx new file mode 100644 index 00000000000..811202c4209 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.integration.test.tsx @@ -0,0 +1,57 @@ +import { act, render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it, vi } from "vitest"; +import TopAgents from "./TopAgents"; +import type { TagSummaryRow } from "./agentCatalog"; + +const desktopTag = "User-Agent: claude-cli/2.1.263 (external, claude-desktop-3p)"; +const row = (tag: string, tokens: number): TagSummaryRow => ({ + tag, + total_spend: tokens / 100, + total_tokens: tokens, + total_requests: 1, + successful_requests: 1, + failed_requests: 0, + unique_users: 1, +}); +const rows = [row(desktopTag, 400), row("User-Agent: claude-code/2.1.263", 200)]; + +describe("TopAgents row navigation", () => { + it("should render recognizable client logos and open usage when the row totals are clicked", async () => { + const user = userEvent.setup(); + const onOpenAgent = vi.fn(); + render(); + + expect(screen.getByRole("heading", { name: "Top agents" })).toBeInTheDocument(); + const desktop = screen.getByRole("button", { name: "View Claude Desktop usage" }); + expect(within(desktop).getByRole("img", { name: "Claude Desktop logo" })).toHaveAttribute( + "src", + expect.any(String), + ); + expect(screen.getByRole("img", { name: "Claude Code logo" })).toBeInTheDocument(); + + await user.click(within(desktop).getByText("400 tokens")); + expect(onOpenAgent).toHaveBeenCalledOnce(); + expect(onOpenAgent).toHaveBeenCalledWith(expect.objectContaining({ id: "claude-desktop", tags: [desktopTag] })); + }); + + it("should open a focused agent row with the keyboard", async () => { + const user = userEvent.setup(); + const onOpenAgent = vi.fn(); + render(); + act(() => screen.getByRole("button", { name: "View Claude Desktop usage" }).focus()); + + await user.keyboard("{Enter}"); + expect(onOpenAgent).toHaveBeenCalledWith(expect.objectContaining({ id: "claude-desktop" })); + }); + + it("should leave an unfilterable residual descriptive instead of making it an inaccurate drill", () => { + const rollup = { ...row("User-Agent: claude-cli", 1000), total_requests: 2 }; + render(); + + expect(screen.getByText("Claude (unspecified)")).toBeInTheDocument(); + expect(screen.getByText("Client not identified")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "View Claude (unspecified) usage" })).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "View Claude Desktop usage" })).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.tsx index be47b96f3f3..2e3575639e3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/TopAgents.tsx @@ -1,7 +1,7 @@ "use client"; import React, { useMemo, useState } from "react"; -import { ArrowRight, Bot } from "lucide-react"; +import { ArrowRight, Bot, Monitor, Terminal } from "lucide-react"; import { StackedUsageChart } from "@/components/shared/charts"; import { Logo } from "@/components/molecules/logo/Logo"; import type { DailyData } from "@/components/UsagePage/types"; @@ -34,33 +34,13 @@ const MONOGRAMS: Readonly> = node: { text: "JS", className: "bg-[#3c873a] text-white" }, curl: { text: "cu", className: "bg-[#073551] text-white" }, }; +const CLIENT_SURFACES: Readonly>> = { + "claude-desktop": Monitor, + "claude-code": Terminal, +}; -/** Claude Code's mark: a white spark on Claude terracotta. Drawn here since no Claude asset ships in the repo. */ -function ClaudeSpark() { - const rays = Array.from({ length: 12 }, (_, i) => i * 30); - return ( - - - - ); -} - -function AgentGlyph({ agent }: { agent: AgentRow }) { - if (agent.id === "claude-code") return ; - if (agent.logo) return ; +function AgentGlyph({ agent }: { agent: Pick }) { + if (agent.logo) return ; const monogram = MONOGRAMS[agent.id]; if (monogram) { return ( @@ -77,10 +57,16 @@ function AgentGlyph({ agent }: { agent: AgentRow }) { return {agent.label.charAt(0)}; } -function AgentMark({ agent }: { agent: AgentRow }) { +export function AgentMark({ agent }: { agent: Pick }) { + const Surface = CLIENT_SURFACES[agent.id]; return ( - + + {Surface && ( + + + )} ); } @@ -98,8 +84,9 @@ function AgentItem({ color: string | undefined; onOpen?: (agent: AgentRow) => void; }) { - return ( -
  • + const openRow = onOpen && agent.tags.length > 0 ? () => onOpen(agent) : undefined; + const content = ( + <> {rank}. @@ -113,15 +100,11 @@ function AgentItem({ {agent.label} - {onOpen ? ( - + ) : ( {agent.description} )} @@ -132,6 +115,27 @@ function AgentItem({ {formatCompactUsd(agent.spend)} · {share.toFixed(1)}% + + ); + const rowClassName = + "group grid w-full grid-cols-[1.5rem_auto_minmax(0,1fr)_auto] items-center gap-3 py-2.5 text-left"; + return ( +
  • + {openRow ? ( + + ) : ( +
    {content}
    + )}
  • ); } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageBreakdownChart.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageBreakdownChart.integration.test.tsx new file mode 100644 index 00000000000..bedc748ab44 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageBreakdownChart.integration.test.tsx @@ -0,0 +1,80 @@ +import { render, screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import UsageBreakdownChart from "./UsageBreakdownChart"; + +const metrics: SpendMetrics = { + spend: 9, + total_tokens: 800, + prompt_tokens: 600, + completion_tokens: 200, + api_requests: 3, + successful_requests: 3, + failed_requests: 0, + cache_read_input_tokens: 0, + cache_creation_input_tokens: 0, +}; +const results: DailyData[] = [ + { + date: "2026-10-01", + metrics, + breakdown: { + model_groups: { "claude-sonnet-4": { metrics, metadata: {}, api_key_breakdown: {} } }, + models: {}, + mcp_servers: {}, + providers: {}, + api_keys: {}, + entities: {}, + }, + }, +]; + +describe("UsageBreakdownChart", () => { + it("should render the current Usage chart controls and model leaderboard", () => { + render(); + + expect(screen.getByRole("heading", { name: "Daily usage" })).toBeInTheDocument(); + expect(screen.getByRole("heading", { name: "Top models" })).toBeInTheDocument(); + expect(screen.getByRole("radio", { name: "Spend" })).toBeChecked(); + expect(screen.getByRole("radio", { name: "Daily" })).toBeChecked(); + expect(screen.getByRole("radio", { name: "Linear" })).toBeChecked(); + expect(screen.getByText("claude-sonnet-4")).toBeInTheDocument(); + expect(within(screen.getByRole("listitem")).getByText("$9.00")).toBeInTheDocument(); + }); + + it("should keep metric, time bucket and scale controls working in the compact panel", async () => { + const user = userEvent.setup(); + const deploymentResults = results.map((day) => ({ + ...day, + breakdown: { ...day.breakdown, model_groups: {}, models: day.breakdown.model_groups }, + })); + render(); + + expect(screen.getByText("claude-sonnet-4")).toBeInTheDocument(); + + await user.click(screen.getByRole("radio", { name: "Tokens" })); + expect(screen.getByText("Daily tokens by model (top 8, rest grouped as Other)")).toBeInTheDocument(); + expect(within(screen.getByRole("listitem")).getByText("800")).toBeInTheDocument(); + expect(screen.queryByText("$9.00")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("radio", { name: "Weekly" })); + expect(screen.getByRole("heading", { name: "Weekly usage" })).toBeInTheDocument(); + expect(screen.getByText("Weekly tokens by model (top 8, rest grouped as Other)")).toBeInTheDocument(); + + await user.click(screen.getByRole("radio", { name: "Log" })); + expect(screen.getByRole("radio", { name: "Log" })).toBeChecked(); + + await user.click(screen.getByRole("radio", { name: "Requests" })); + expect(screen.getByText("Weekly requests by model (top 8, rest grouped as Other)")).toBeInTheDocument(); + expect(within(screen.getByRole("listitem")).getByText("3")).toBeInTheDocument(); + expect(screen.queryByText("800")).not.toBeInTheDocument(); + }); + + it("should show loading rather than an empty leaderboard until the data has settled", () => { + render(); + + expect(screen.getByRole("status", { name: "Loading top models" })).toBeInTheDocument(); + expect(screen.queryByText("No usage in this range")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageBreakdownChart.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageBreakdownChart.tsx new file mode 100644 index 00000000000..377a3895b1c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageBreakdownChart.tsx @@ -0,0 +1,106 @@ +"use client"; + +import { useMemo, useState } from "react"; +import type { DailyData } from "@/components/UsagePage/types"; +import { StackedUsageChart, type StackedUsageScale } from "@/components/shared/charts"; +import { cn } from "@/lib/cva.config"; +import { BreakdownControls, Leaderboard, useBreakdown, type BreakdownState } from "./BreakdownChart"; +import { + bucketSeries, + bucketTotals, + formatMetricValue, + labelForDate, + modelColorMap, + type BreakdownDimension, + type Granularity, + type UsageMetric, +} from "./overviewData"; +import { ChartSkeleton, Panel, PANEL_INSET_X, Segmented } from "./Primitives"; + +const METRIC_NOUN: Record = { spend: "spend", tokens: "tokens", requests: "requests" }; +const GRANULARITY_OPTIONS = [ + { value: "day", label: "Daily" }, + { value: "week", label: "Weekly" }, +] as const satisfies readonly { value: Granularity; label: string }[]; +const SCALE_OPTIONS = [ + { value: "linear", label: "Linear" }, + { value: "log", label: "Log" }, +] as const satisfies readonly { value: StackedUsageScale; label: string }[]; + +interface UsageBreakdownChartProps { + results: readonly DailyData[]; + loading: boolean; + compact?: boolean; + colorByModel?: ReadonlyMap; + dimension?: BreakdownDimension; +} + +export default function UsageBreakdownChart({ + results, + loading, + compact = false, + colorByModel, + dimension = "model_groups", +}: UsageBreakdownChartProps) { + const [metric, setMetric] = useState("spend"); + const state: BreakdownState = { metric, dimension }; + const [granularity, setGranularity] = useState("day"); + const [scale, setScale] = useState("linear"); + const modelColors = useMemo(() => colorByModel ?? modelColorMap(results), [colorByModel, results]); + const { series: dailySeries, ranking } = useBreakdown(results, state, 8, modelColors); + const series = useMemo(() => bucketSeries(dailySeries, granularity), [dailySeries, granularity]); + const totalsByBucket = useMemo(() => bucketTotals(series), [series]); + const format = (value: number) => formatMetricValue(value, state.metric); + + return ( + + setMetric(next.metric)} showDimension={false} /> + + + + } + bodyClassName="px-0 pt-4 pb-0" + > +
    + {loading ? ( + + ) : ( + labelForDate(series, date)} + scale={scale} + format={format} + totalFor={(date) => totalsByBucket.get(date)} + className={compact ? "h-56" : undefined} + /> + )} +
    +
    +

    Top models

    + {loading ? ( +
    + +
    + ) : ( + + )} +
    +
    + ); +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageOverview.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageOverview.tsx index afd6e5ba48b..4b6fa4ee324 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageOverview.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageOverview.tsx @@ -6,23 +6,13 @@ * matching colors. */ -import { type ReactNode, useMemo, useState } from "react"; +import { type ReactNode } from "react"; import type { DailyData } from "@/components/UsagePage/types"; import { KeyRound, Users } from "lucide-react"; -import { StackedUsageChart, type StackedUsageScale } from "@/components/shared/charts"; -import { - bucketSeries, - bucketTotals, - labelForDate, - formatMetricValue, - type Granularity, - type OverviewTotals, - type UsageMetric, -} from "./overviewData"; -import { BreakdownControls, Leaderboard, useBreakdown, type BreakdownState } from "./BreakdownChart"; -import { cn } from "@/lib/cva.config"; -import { ChartSkeleton, Panel, PANEL_INSET_X, Segmented } from "./Primitives"; +import type { OverviewTotals } from "./overviewData"; +import { Panel } from "./Primitives"; import UsageStatStrip from "./UsageStatStrip"; +import UsageBreakdownChart from "./UsageBreakdownChart"; interface UsageOverviewProps { results: readonly DailyData[]; @@ -37,16 +27,6 @@ interface UsageOverviewProps { providerBreakdown: ReactNode; } -const METRIC_NOUN: Record = { spend: "spend", tokens: "tokens", requests: "requests" }; -const GRANULARITY_OPTIONS = [ - { value: "day", label: "Daily" }, - { value: "week", label: "Weekly" }, -] as const satisfies readonly { value: Granularity; label: string }[]; -const SCALE_OPTIONS = [ - { value: "linear", label: "Linear" }, - { value: "log", label: "Log" }, -] as const satisfies readonly { value: StackedUsageScale; label: string }[]; - export default function UsageOverview({ results, totals, @@ -59,14 +39,6 @@ export default function UsageOverview({ providerBreakdown, budget, }: UsageOverviewProps) { - const [state, setState] = useState({ metric: "spend", dimension: "model_groups" }); - const [granularity, setGranularity] = useState("day"); - const [scale, setScale] = useState("linear"); - const { series: dailySeries, ranking } = useBreakdown(results, state, 8); - const series = useMemo(() => bucketSeries(dailySeries, granularity), [dailySeries, granularity]); - const totalsByBucket = useMemo(() => bucketTotals(series), [series]); - const format = (value: number) => formatMetricValue(value, state.metric); - return (
    - - - - - - } - bodyClassName="px-0 pt-4 pb-0" - > -
    - {loading ? ( - - ) : ( - labelForDate(series, date)} - scale={scale} - format={format} - totalFor={(date) => totalsByBucket.get(date)} - /> - )} -
    -
    -

    Top models

    - -
    -
    + {topAgents} {providerBreakdown} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.integration.test.tsx new file mode 100644 index 00000000000..2e2c42cb218 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.integration.test.tsx @@ -0,0 +1,60 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; +import type { OverviewTotals } from "./overviewData"; +import UsageStatStrip from "./UsageStatStrip"; + +const totals: OverviewTotals = { + spend: 42.5, + requests: 10, + successful: 8, + failed: 2, + tokens: 9123456, + inputTokens: 8123456, + outputTokens: 1000000, + cacheReadTokens: 5123456, + cacheWriteTokens: 2000000, + avgCostPerRequest: 4.25, + avgLatencyMs: 250, + successRate: 80, +}; + +const defaultProps = { + results: [], + totals, + loading: false, + requestCountsPending: false, + budget: null, +}; + +describe("UsageStatStrip token breakdown", () => { + it("should render the existing totals and open the full token counts", async () => { + const user = userEvent.setup(); + render(); + + expect(screen.getByText("Total Tokens")).toBeInTheDocument(); + expect(screen.getByText("ok")).toBeInTheDocument(); + expect(screen.getByText("failed")).toBeInTheDocument(); + expect(screen.getByText("80.0% success rate")).toBeInTheDocument(); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "View token breakdown" })); + const breakdown = await screen.findByRole("dialog", { name: "What counts toward total tokens?" }); + expect(breakdown).toHaveTextContent(/Input tokens\s*8,123,456/); + expect(breakdown).toHaveTextContent(/Cache reads\s*5,123,456/); + expect(breakdown).toHaveTextContent(/Cache writes\s*2,000,000/); + expect(breakdown).toHaveTextContent(/Output tokens\s*1,000,000/); + expect(breakdown).toHaveTextContent(/Total tokens\s*9,123,456/); + expect(breakdown).toHaveTextContent("Input includes cache reads and writes."); + expect(breakdown).toHaveTextContent("Compare the same person, client and time range"); + }); + + it("should hide the token breakdown and request counts while usage is loading", () => { + render(); + + expect(screen.getByText("Total Tokens")).toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "View token breakdown" })).not.toBeInTheDocument(); + expect(screen.queryByText("ok")).not.toBeInTheDocument(); + expect(screen.queryByText("failed")).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.tsx index 318b7758370..5d9f8d6e0e3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/UsageStatStrip.tsx @@ -3,6 +3,7 @@ import React, { useMemo } from "react"; import type { DailyData } from "@/components/UsagePage/types"; import { cn } from "@/lib/cva.config"; +import { Popover, PopoverContent, PopoverTitle, PopoverTrigger } from "@/components/ui/popover"; import { dailyTotals, formatCompact, formatLatency, formatUsd, type OverviewTotals } from "./overviewData"; import { PANEL_INSET_X, Sparkline, Stat } from "./Primitives"; @@ -59,6 +60,46 @@ export default function UsageStatStrip({ pending={loading} hint={loading ? undefined : `${formatCompact(totals.cacheReadTokens)} cache read`} /> + {!loading && ( + + + View token breakdown + + + What counts toward total tokens? +
    +
    +
    Input tokens
    +
    {totals.inputTokens.toLocaleString()}
    +
    +
    +
    Cache reads
    +
    {totals.cacheReadTokens.toLocaleString()}
    +
    +
    +
    Cache writes
    +
    {totals.cacheWriteTokens.toLocaleString()}
    +
    +
    +
    Output tokens
    +
    {totals.outputTokens.toLocaleString()}
    +
    +
    +
    Total tokens
    +
    {totals.tokens.toLocaleString()}
    +
    +
    +

    + Input includes cache reads and writes. Cached input has separate pricing, so total tokens alone cannot + explain the cost. +

    +

    + A client may show a different subset of tokens. Compare the same person, client and time range before + comparing counts. +

    +
    +
    + )}
    diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/agentCatalog.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/agentCatalog.test.ts index 14ad625c79f..174539c0a1f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/agentCatalog.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/agentCatalog.test.ts @@ -1,5 +1,14 @@ import { describe, expect, it } from "vitest"; -import { agentDailyData, isUserAgentTag, topAgents, userAgentProduct, type TagSummaryRow } from "./agentCatalog"; +import { + agentDailyData, + isUserAgentTag, + parseUsageClient, + topAgents, + usageClientFromUserAgent, + userAgentProduct, + UsageClient, + type TagSummaryRow, +} from "./agentCatalog"; import type { DailyData } from "@/components/UsagePage/types"; const metric = (spend: number, tokens: number) => ({ @@ -33,6 +42,26 @@ const row = ( unique_users: users, }); +describe("client identification", () => { + it("should accept only supported client IDs at a selection boundary", () => { + expect(parseUsageClient("claude-desktop")).toBe(UsageClient.ClaudeDesktop); + expect(parseUsageClient("claude-code")).toBe(UsageClient.ClaudeCode); + expect(parseUsageClient("claude-cli")).toBeNull(); + expect(parseUsageClient(null)).toBeNull(); + }); + + it.each<[string, UsageClient | null]>([ + [" User Agent: claude-desktop/1.0 ", UsageClient.ClaudeDesktop], + ["claude-cli/1.0 (external, cli)", UsageClient.ClaudeCode], + ["claude-cli/1.0 (external, cli, claude-desktop-3p)", UsageClient.ClaudeDesktop], + ["python-httpx/1.0 (claude-desktop-3p)", null], + ["claude-cli/1.0 (integration=claude-desktop-3p)", null], + ["claude-cli", null], + ])("should classify the full user agent %s without matching incidental markers", (agent, expected) => { + expect(usageClientFromUserAgent(agent)).toBe(expected); + }); +}); + describe("userAgentProduct", () => { it("drops the prefix, version and platform detail", () => { expect(userAgentProduct("User-Agent: claude-cli/2.1.263 (external, cli)")).toBe("claude-cli"); @@ -79,13 +108,14 @@ describe("topAgents", () => { expect(agents[0].logo).toBeTruthy(); }); - it("maps claude-cli to Claude Code and ranks by tokens", () => { + it("should keep a bare claude-cli rollup unspecified and rank by tokens", () => { const agents = topAgents([ row("User-Agent: python", 600, 300), row("User-Agent: claude-cli", 370, 343), row("User-Agent: Cursor/1.0", 1, 7), ]); - expect(agents.map((agent) => agent.label)).toEqual(["Claude Code", "Python", "Cursor"]); + expect(agents.map((agent) => agent.label)).toEqual(["Claude (unspecified)", "Python", "Cursor"]); + expect(agents[0]).toMatchObject({ description: "Client not identified", tags: ["User-Agent: claude-cli"] }); expect(agents[0].kind).toBe("agent"); expect(agents[1].kind).toBe("sdk"); }); @@ -138,7 +168,10 @@ describe("topAgents", () => { row("User-Agent: claude-cli/2.1.263 (external, cli)", 10, 100), row("User-Agent: claude-code", 2, 20), ]); - expect([...agent.tags].sort()).toEqual(["User-Agent: claude-cli", "User-Agent: claude-code"]); + expect([...agent.tags].sort()).toEqual([ + "User-Agent: claude-cli/2.1.263 (external, cli)", + "User-Agent: claude-code", + ]); expect(agent).toMatchObject({ spend: 12, tokens: 120 }); }); @@ -152,6 +185,100 @@ describe("topAgents", () => { "User-Agent: opencode/1.18.30 ai-sdk", ]); }); + + it("should split Desktop and Code from full user agents without counting their bare rollup again", () => { + const desktop = row("User-Agent: claude-cli/2.1.263 (external, claude-desktop-3p)", 4, 400, { requests: 2 }); + const code = row("User-Agent: claude-cli/2.1.264 (external, cli)", 6, 600, { requests: 3 }); + const agents = topAgents([row("User-Agent: claude-cli", 10, 1000, { requests: 5 }), desktop, code]); + const expectedDesktop = { + label: "Claude Desktop", + spend: 4, + tokens: 400, + requests: 2, + tags: [desktop.tag], + }; + const expectedCode = { + label: "Claude Code", + spend: 6, + tokens: 600, + requests: 3, + tags: [code.tag], + }; + expect(agents).toHaveLength(2); + expect(agents.find((agent) => agent.id === "claude-desktop")).toMatchObject(expectedDesktop); + expect(agents.find((agent) => agent.id === "claude-code")).toMatchObject(expectedCode); + }); + + it("should identify Desktop before cli markers and keep unmarked versions unspecified without a rollup", () => { + const desktopTag = "User-Agent: CLAUDE-CLI/2.1.263 (external, cli, claude-desktop-3p)"; + const ambiguousTag = "User-Agent: claude-cli/2.1.264"; + const agents = topAgents([row(desktopTag, 4, 400), row(ambiguousTag, 6, 600)]); + expect(agents).toHaveLength(2); + expect(agents.find((agent) => agent.id === "claude-desktop")).toMatchObject({ + spend: 4, + tokens: 400, + tags: [desktopTag], + }); + expect(agents.find((agent) => agent.id === "claude-unspecified")).toMatchObject({ + spend: 6, + tokens: 600, + tags: [ambiguousTag], + }); + }); + + it("should not treat an unknown cli-like marker as proof of Claude Code", () => { + const tag = "User-Agent: claude-cli/2.1.263 (external, custom-cli)"; + const [agent] = topAgents([row(tag, 4, 400)]); + expect(agent).toMatchObject({ id: "claude-unspecified", tags: [tag] }); + }); + + it("should retain missing version usage as unspecified without offering an inaccurate bare-tag filter", () => { + const agents = topAgents([ + row("User-Agent: claude-cli", 10, 1000, { requests: 5, users: 4 }), + row("User-Agent: claude-cli/2.1.263 (external, claude-desktop-3p)", 4, 400, { requests: 2, users: 2 }), + row("User-Agent: claude-cli/2.1.264", 2, 200, { requests: 1, users: 1 }), + ]); + const expectedUnspecified = { + spend: 6, + tokens: 600, + requests: 3, + users: 1, + tags: [], + }; + expect(agents).toHaveLength(2); + expect(agents.find((agent) => agent.id === "claude-desktop")).toMatchObject({ spend: 4, tokens: 400, requests: 2 }); + expect(agents.find((agent) => agent.id === "claude-unspecified")).toMatchObject(expectedUnspecified); + }); + + it.each([ + { spend: 11, tokens: 1000, requests: 5 }, + { spend: 10, tokens: 1001, requests: 5 }, + { spend: 10, tokens: 1000, requests: 6 }, + ])("should keep the rollup unspecified when version totals exceed it (%j)", ({ spend, tokens, requests }) => { + const rollup = row("User-Agent: claude-cli", 10, 1000, { requests: 5 }); + const agents = topAgents([ + rollup, + row("User-Agent: claude-cli/2.1.263 (external, claude-desktop-3p)", spend, tokens, { requests }), + ]); + const expectedRollup = { + id: "claude-unspecified", + spend: 10, + tokens: 1000, + requests: 5, + tags: [rollup.tag], + }; + expect(agents).toHaveLength(1); + expect(agents[0]).toMatchObject(expectedRollup); + }); + + it("should avoid a spurious unspecified client from floating-point cost sums", () => { + const agents = topAgents([ + row("User-Agent: claude-cli", 0.3, 300, { requests: 2 }), + row("User-Agent: claude-cli/2.1.263 (external, claude-desktop-3p)", 0.1, 100), + row("User-Agent: claude-cli/2.1.264 (external, cli)", 0.2, 200), + ]); + expect(agents.map((agent) => agent.id).sort()).toEqual(["claude-code", "claude-desktop"]); + }); }); describe("agentDailyData", () => { @@ -180,4 +307,35 @@ describe("agentDailyData", () => { expect(result.breakdown.models["Claude Code"].metrics).toMatchObject({ spend: 10, total_tokens: 100 }); expect(result.breakdown.models["Codex"].metrics).toMatchObject({ spend: 4, total_tokens: 40 }); }); + + it("should preserve Desktop attribution and unspecified residual in the daily chart", () => { + const day = { + date: "2026-10-01", + metrics: metric(10, 1000).metrics, + breakdown: { + models: {}, + model_groups: {}, + mcp_servers: {}, + providers: {}, + api_keys: {}, + entities: { + "User-Agent: claude-cli": { metrics: { ...metric(10, 1000).metrics, api_requests: 2 } }, + "User-Agent: claude-cli/2.1.263 (external, claude-desktop-3p)": metric(4, 400), + }, + }, + } as unknown as DailyData; + const [result] = agentDailyData([day]); + expect(result.metrics).toMatchObject({ spend: 10, total_tokens: 1000, api_requests: 2 }); + expect(Object.keys(result.breakdown.models).sort()).toEqual(["Claude (unspecified)", "Claude Desktop"]); + expect(result.breakdown.models["Claude Desktop"].metrics).toMatchObject({ + spend: 4, + total_tokens: 400, + api_requests: 1, + }); + expect(result.breakdown.models["Claude (unspecified)"].metrics).toMatchObject({ + spend: 6, + total_tokens: 600, + api_requests: 1, + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/agentCatalog.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/agentCatalog.ts index 57f481c14be..e4fe94791fb 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/agentCatalog.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/agentCatalog.ts @@ -1,4 +1,5 @@ import anthropicLogo from "../../../../../../../public/assets/logos/anthropic.svg"; +import claudeLogo from "../../../../../../../public/assets/logos/claude.svg"; import cursorLogo from "../../../../../../../public/assets/logos/cursor.svg"; import copilotLogo from "../../../../../../../public/assets/logos/github_copilot.svg"; import githubLogo from "../../../../../../../public/assets/logos/github.svg"; @@ -18,6 +19,17 @@ export type TagSummaryRow = components["schemas"]["TagSummaryMetrics"]; export type AgentKind = "agent" | "sdk"; +export enum UsageClient { + ClaudeDesktop = "claude-desktop", + ClaudeCode = "claude-code", +} + +export const parseUsageClient = (value: string | null): UsageClient | null => { + if (value === UsageClient.ClaudeDesktop) return UsageClient.ClaudeDesktop; + if (value === UsageClient.ClaudeCode) return UsageClient.ClaudeCode; + return null; +}; + export interface AgentRow { id: string; label: string; @@ -45,17 +57,32 @@ interface AgentFamily { /** * Most specific first. Coding agents beat the SDK they are built on, so `codex_sdk_ts` lands on * Codex and only a bare `OpenAI/...` client falls through to the OpenAI SDK. Codex wears OpenAI's - * mark as in the Lens trace view; Claude Code gets its own spark tile rather than Anthropic's wordmark. + * mark as in the Lens trace view; Claude clients use the bundled Claude brand mark. */ const FAMILIES: readonly AgentFamily[] = [ { - id: "claude-code", + id: UsageClient.ClaudeDesktop, + label: "Claude Desktop", + description: "Anthropic's desktop assistant", + kind: "agent", + logo: claudeLogo.src, + match: /^claude[-_ ]desktop\b|^claude-cli\/[^\s]+\s+\((?:[^)]*,\s*)?claude-desktop-3p\s*[,)]/i, + }, + { + id: UsageClient.ClaudeCode, label: "Claude Code", description: "Anthropic's coding agent", kind: "agent", - // No Claude asset ships in the repo, so TopAgents draws the spark tile itself. - logo: null, - match: /^(claude-cli|claude-code|claude_code|claude-user)\b/i, + logo: claudeLogo.src, + match: /^(claude-code|claude_code|claude-user)\b|^claude-cli\/[^\s]+\s+\((?:[^)]*,\s*)?cli\s*[,)]/i, + }, + { + id: "claude-unspecified", + label: "Claude (unspecified)", + description: "Client not identified", + kind: "agent", + logo: claudeLogo.src, + match: /^claude-cli\b/i, }, { id: "codex", @@ -210,6 +237,9 @@ export const userAgentProduct = (tag: string): string => { const familyFor = (product: string): AgentFamily | null => FAMILIES.find((family) => family.match.test(product)) ?? null; +export const usageClientFromUserAgent = (userAgent: string): UsageClient | null => + parseUsageClient(familyFor(userAgent.trim().replace(USER_AGENT_PREFIX, ""))?.id ?? null); + const isBareTag = (row: TagSummaryRow, product: string) => row.tag.trim().replace(USER_AGENT_PREFIX, "").trim() === product; @@ -246,9 +276,44 @@ const mergeAgents = (a: AgentRow, b: AgentRow): AgentRow => ({ tokens: a.tokens + b.tokens, requests: a.requests + b.requests, users: Math.max(a.users, b.users), - tags: [...a.tags, ...b.tags], + tags: a.tags.length && b.tags.length ? [...a.tags, ...b.tags] : [], }); +const agentsForProduct = (product: string, rows: readonly TagSummaryRow[]): AgentRow[] => { + if (product.toLowerCase() !== "claude-cli") return [agentFor(product, contributingRows(product, rows))]; + const bare = rows.find((row) => isBareTag(row, product)); + const versions = rows.filter((row) => !isBareTag(row, product)); + const versionAgents = versions.map((row) => agentFor(row.tag.trim().replace(USER_AGENT_PREFIX, ""), [row])); + if (!bare) return versionAgents; + const rollup = agentFor(product, [bare]); + if (!versions.length) return [rollup]; + const emptyTotals = { spend: 0, tokens: 0, requests: 0, users: 0 }; + const totals = versionAgents.reduce( + (sum, agent) => ({ + spend: sum.spend + agent.spend, + tokens: sum.tokens + agent.tokens, + requests: sum.requests + agent.requests, + users: sum.users + agent.users, + }), + emptyTotals, + ); + if (totals.spend > rollup.spend + 1e-9 || totals.tokens > rollup.tokens || totals.requests > rollup.requests) { + return [rollup]; + } + const residual = { + ...rollup, + spend: Math.max(0, rollup.spend - totals.spend), + tokens: rollup.tokens - totals.tokens, + requests: rollup.requests - totals.requests, + users: Math.max(0, rollup.users - totals.users), + // A bare tag also selects the identified clients, so it cannot filter the residual alone. + tags: [], + }; + return residual.spend > 1e-9 || residual.tokens > 0 || residual.requests > 0 + ? [...versionAgents, residual] + : versionAgents; +}; + /** * One row per agent from `/tag/summary`. Products are keyed case-sensitively: prod reports `python` * and `Python` as distinct products, and folding them let one rollup overwrite the other. @@ -256,13 +321,10 @@ const mergeAgents = (a: AgentRow, b: AgentRow): AgentRow => ({ export const topAgents = (rows: readonly TagSummaryRow[]): AgentRow[] => { const userAgentRows = rows.filter((row) => isUserAgentTag(row.tag) && userAgentProduct(row.tag)); const products = [...new Set(userAgentRows.map((row) => userAgentProduct(row.tag)))]; - const perProduct = products.map((product) => - agentFor( + const perProduct = products.flatMap((product) => + agentsForProduct( product, - contributingRows( - product, - userAgentRows.filter((row) => userAgentProduct(row.tag) === product), - ), + userAgentRows.filter((row) => userAgentProduct(row.tag) === product), ), ); const byAgent = perProduct.reduce>((acc, agent) => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/clientAgentDetail.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/clientAgentDetail.test.ts new file mode 100644 index 00000000000..bf44b18e2f3 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/clientAgentDetail.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import type { AgentRow } from "./agentCatalog"; +import { clientAgentDetail } from "./clientAgentDetail"; + +const desktopTag = "User-Agent: claude-cli/2.1.1 (external, claude-desktop-3p)"; +const codeTag = "User-Agent: claude-cli/2.1.1 (external, cli)"; +const agent: AgentRow = { + id: "claude-desktop", + label: "Claude Desktop", + description: "Desktop assistant", + kind: "agent", + logo: null, + spend: 3, + tokens: 300, + requests: 3, + users: 2, + tags: [desktopTag], +}; +const oneRequestAgent = { ...agent, spend: 1, tokens: 100, requests: 1 }; +const unknownTokenBreakdown = { inputTokens: null, outputTokens: null, cacheReadTokens: null, cacheWriteTokens: null }; + +const metrics = (overrides: Partial = {}): SpendMetrics => ({ + spend: 1, + prompt_tokens: 80, + completion_tokens: 20, + total_tokens: 100, + api_requests: 1, + successful_requests: 1, + failed_requests: 0, + cache_read_input_tokens: 30, + cache_creation_input_tokens: 10, + ...overrides, +}); + +const day = (date: string, entities: Record): DailyData => ({ + date, + metrics: metrics({ spend: 999, total_tokens: 999_999, api_requests: 999 }), + breakdown: { + models: { unrelated: { metrics: metrics({ spend: 999 }), metadata: {}, api_key_breakdown: {} } }, + model_groups: {}, + providers: {}, + api_keys: {}, + mcp_servers: {}, + entities: Object.fromEntries( + Object.entries(entities).map(([tag, value]) => [tag, { metrics: value, metadata: {}, api_key_breakdown: {} }]), + ), + }, +}); + +describe("clientAgentDetail", () => { + it("should preserve the clicked summary before daily data arrives", () => { + const expected = { + summary: { spend: 3, tokens: 300, requests: 3 }, + dailyMatchesSummary: false, + tokenBreakdown: unknownTokenBreakdown, + }; + expect(clientAgentDetail(agent, [])).toEqual(expected); + }); + + it("should reconcile exact contributing tags once without global traffic", () => { + const secondTag = "User-Agent: claude-desktop/2.0"; + const result = clientAgentDetail({ ...agent, tags: [desktopTag, secondTag, desktopTag] }, [ + day("2026-10-22", { [desktopTag]: metrics(), [secondTag]: metrics(), [codeTag]: metrics({ spend: 50 }) }), + day("2026-10-20", { [desktopTag]: metrics(), "User-Agent: claude-cli": metrics({ spend: 500 }) }), + day("2026-10-21", { [codeTag]: metrics() }), + ]); + + expect(result.dailyMatchesSummary).toBe(true); + const expectedTokens = { + inputTokens: 240, + outputTokens: 60, + cacheReadTokens: 90, + cacheWriteTokens: 30, + }; + expect(result.tokenBreakdown).toEqual(expectedTokens); + }); + + it("should keep missing cache counts unknown without hiding recorded token counts", () => { + const result = clientAgentDetail(oneRequestAgent, [ + day("2026-10-22", { [desktopTag]: metrics({ cache_read_input_tokens: undefined }) }), + ]); + + const expectedTokens = { + inputTokens: 80, + outputTokens: 20, + cacheReadTokens: null, + cacheWriteTokens: null, + }; + expect(result.tokenBreakdown).toEqual(expectedTokens); + }); + + it("should distinguish recorded zero caching from missing counts", () => { + const result = clientAgentDetail(oneRequestAgent, [ + day("2026-10-22", { + [desktopTag]: metrics({ cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }), + }), + ]); + + expect(result.tokenBreakdown.cacheReadTokens).toBe(0); + expect(result.tokenBreakdown.cacheWriteTokens).toBe(0); + }); + + it("should preserve summary totals but withhold token breakdown when daily snapshots disagree", () => { + const result = clientAgentDetail(agent, [day("2026-10-22", { [desktopTag]: metrics() })]); + + expect(result.summary).toEqual({ spend: 3, tokens: 300, requests: 3 }); + expect(result.dailyMatchesSummary).toBe(false); + expect(result.tokenBreakdown).toEqual(unknownTokenBreakdown); + }); + + it("should not turn unrelated tags or ambiguous residuals into zero client activity", () => { + const unrelated = clientAgentDetail(agent, [day("2026-10-22", { [codeTag]: metrics() })]); + const residual = clientAgentDetail({ ...agent, tags: [] }, [day("2026-10-22", { [desktopTag]: metrics() })]); + + expect(unrelated.tokenBreakdown).toEqual(unknownTokenBreakdown); + expect(residual.tokenBreakdown).toEqual(unknownTokenBreakdown); + expect(residual.summary.spend).toBe(agent.spend); + }); + + it("should allow floating-point rounding without dropping known breakdowns", () => { + const decimalAgent = { ...agent, spend: 0.3, tokens: 200, requests: 2 }; + const result = clientAgentDetail(decimalAgent, [ + day("2026-10-21", { [desktopTag]: metrics({ spend: 0.1 }) }), + day("2026-10-22", { [desktopTag]: metrics({ spend: 0.2 }) }), + ]); + + expect(result.dailyMatchesSummary).toBe(true); + expect(result.tokenBreakdown.inputTokens).toBe(160); + }); + + it.each([ + { prompt_tokens: 100 }, + { cache_read_input_tokens: 80, cache_creation_input_tokens: 10 }, + { cache_read_input_tokens: Number.NaN }, + { cache_read_input_tokens: Number.POSITIVE_INFINITY }, + { cache_read_input_tokens: -1 }, + ])("should suppress cache counts that do not reconcile (%o)", (invalidCounts) => { + const result = clientAgentDetail(oneRequestAgent, [day("2026-10-22", { [desktopTag]: metrics(invalidCounts) })]); + + expect(result.tokenBreakdown.cacheReadTokens).toBeNull(); + expect(result.tokenBreakdown.cacheWriteTokens).toBeNull(); + }); + + it("should withhold non-finite daily counts while preserving the known summary", () => { + const result = clientAgentDetail(agent, [ + day("2026-10-22", { [desktopTag]: metrics({ total_tokens: Number.NaN }) }), + ]); + + expect(result.dailyMatchesSummary).toBe(false); + expect(result.summary.tokens).toBe(300); + expect(result.tokenBreakdown.inputTokens).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/clientAgentDetail.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/clientAgentDetail.ts new file mode 100644 index 00000000000..6c496ece2e9 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/clientAgentDetail.ts @@ -0,0 +1,46 @@ +import type { DailyData, SpendMetrics } from "@/components/UsagePage/types"; +import type { AgentRow } from "./agentCatalog"; +import { reconciledTokenBreakdown, type TokenBreakdown } from "./tokenBreakdown"; + +export interface ClientAgentDetail { + summary: Pick; + dailyMatchesSummary: boolean; + tokenBreakdown: TokenBreakdown; +} + +const knownTotal = (metrics: readonly SpendMetrics[], key: keyof SpendMetrics): number | null => { + const values = metrics.map((entry) => entry[key]); + return values.length > 0 && values.every((value) => typeof value === "number" && Number.isFinite(value)) + ? values.reduce((sum, value) => sum + (value ?? 0), 0) + : null; +}; + +export const clientAgentDetail = (agent: AgentRow, days: readonly DailyData[]): ClientAgentDetail => { + const tags = [...new Set(agent.tags)]; + const metrics = days.flatMap((day) => + tags.flatMap((tag) => { + const entry = day.breakdown.entities[tag]; + return entry ? [entry.metrics] : []; + }), + ); + const spend = knownTotal(metrics, "spend"); + const spendMatches = spend !== null && Math.abs(spend - agent.spend) <= Math.max(1e-8, Math.abs(agent.spend) * 1e-9); + const countsMatch = + knownTotal(metrics, "total_tokens") === agent.tokens && knownTotal(metrics, "api_requests") === agent.requests; + const dailyMatchesSummary = spendMatches && countsMatch; + const tokenTotal = (key: keyof SpendMetrics) => { + const total = dailyMatchesSummary ? knownTotal(metrics, key) : null; + return total !== null && total >= 0 ? total : null; + }; + const inputTokens = tokenTotal("prompt_tokens"); + const outputTokens = tokenTotal("completion_tokens"); + const cacheReadTokens = tokenTotal("cache_read_input_tokens"); + const cacheWriteTokens = tokenTotal("cache_creation_input_tokens"); + const tokenBreakdown = { inputTokens, outputTokens, cacheReadTokens, cacheWriteTokens }; + + return { + summary: { spend: agent.spend, tokens: agent.tokens, requests: agent.requests }, + dailyMatchesSummary, + tokenBreakdown: reconciledTokenBreakdown(agent.tokens, tokenBreakdown), + }; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts index eb4ad2379b7..f00d53ea1bc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.test.ts @@ -1,6 +1,15 @@ import { describe, expect, it } from "vitest"; import type { DailyData } from "@/components/UsagePage/types"; -import { bucketSeries, bucketTotals, labelForDate, rollUpBreakdown, seriesBy } from "./overviewData"; +import { + bucketSeries, + bucketTotals, + colorForModel, + labelForDate, + modelColorMap, + OTHER_COLOR, + rollUpBreakdown, + seriesBy, +} from "./overviewData"; const metrics = (spend: number) => ({ spend, @@ -89,3 +98,86 @@ describe("rollUpBreakdown", () => { expect(JSON.stringify(days)).toBe(before); }); }); + +describe("model colors", () => { + it("should include deployment IDs without replacing the main chart's group colors or guessing aliases", () => { + const result = day("2026-10-01", 12, { "claude-sonnet": 10, "gpt-5": 2 }); + const original = modelColorMap([result]); + result.breakdown.models = { + "claude-sonnet": result.breakdown.model_groups["claude-sonnet"], + "provider-deployment-123": result.breakdown.model_groups["gpt-5"], + }; + const colors = modelColorMap([result]); + const deployments = seriesBy([result], "models", "spend", { top: 8, colorByKey: colors }); + + expect(colors.get("claude-sonnet")).toBe(original.get("claude-sonnet")); + expect(colors.get("gpt-5")).toBe(original.get("gpt-5")); + expect(colors.size).toBe(3); + expect(colors.get("provider-deployment-123")).not.toBe(colors.get("gpt-5")); + expect(deployments.colors).toEqual([colors.get("claude-sonnet"), colors.get("provider-deployment-123")]); + }); + + it("should preserve the overview's existing spend palette and reuse it when a client ranks models differently", () => { + const overview = [day("2026-10-01", 15, { "claude-sonnet": 10, "gpt-5": 4, "gemini-pro": 1 })]; + const client = [day("2026-10-01", 5, { "gpt-5": 4, "claude-sonnet": 1 })]; + const colors = modelColorMap(overview); + const originalSeries = seriesBy(overview, "model_groups", "spend"); + const clientSeries = seriesBy(client, "model_groups", "spend", { top: 8, colorByKey: colors }); + + expect(originalSeries.colors).toEqual(originalSeries.labels.map((model) => colors.get(model))); + expect(clientSeries.labels).toEqual(["gpt-5", "claude-sonnet"]); + expect(clientSeries.colors).toEqual([colors.get("gpt-5"), colors.get("claude-sonnet")]); + expect(clientSeries.colors[0]).not.toBe(originalSeries.colors[0]); + expect(bucketTotals(clientSeries).get("2026-10-01")).toBe(5); + }); + + it("should retain model colors when token ranking and weekly grouping change the chart", () => { + const result = day("2026-10-01", 12, { "claude-sonnet": 10, "gpt-5": 2 }); + result.breakdown.model_groups["claude-sonnet"].metrics.total_tokens = 20; + result.breakdown.model_groups["gpt-5"].metrics.total_tokens = 100; + const colors = modelColorMap([result]); + const tokens = seriesBy([result], "model_groups", "tokens", { top: 8, colorByKey: colors }); + const weekly = bucketSeries(tokens, "week"); + + expect(tokens.labels).toEqual(["gpt-5", "claude-sonnet"]); + expect(tokens.colors).toEqual([colors.get("gpt-5"), colors.get("claude-sonnet")]); + expect(weekly.colors).toEqual(tokens.colors); + expect(bucketTotals(weekly).get("2026-10-01")).toBe(120); + }); + + it("should give a client-only model a deterministic color when it changes rank", () => { + const colors = modelColorMap([day("2026-10-01", 10, { "claude-sonnet": 10 })]); + const first = seriesBy([day("2026-10-01", 10, { "claude-sonnet": 8, "new-model": 2 })], "model_groups", "spend", { + top: 8, + colorByKey: colors, + }); + const second = seriesBy([day("2026-10-01", 10, { "new-model": 8, "claude-sonnet": 2 })], "model_groups", "spend", { + top: 8, + colorByKey: colors, + }); + + expect(first.colors[1]).toBe(second.colors[0]); + expect(first.colors[1]).toBe(colorForModel("new-model", colors)); + expect(colors.has("new-model")).toBe(false); + }); + + it("should keep a model named Other distinct from the grey remainder and retain colors outside the top eight", () => { + const models = { Other: 10, a: 9, b: 8, c: 7, d: 6, e: 5, f: 4, g: 3, h: 2 }; + const results = [day("2026-10-01", 55, models)]; + const colors = modelColorMap(results); + const series = seriesBy(results, "model_groups", "spend", { top: 8, colorByKey: colors }); + const client = seriesBy([day("2026-10-01", 2, { h: 2 })], "model_groups", "spend", { + top: 8, + colorByKey: colors, + }); + + expect(series.labels[0]).toBe("Other"); + expect(series.colors[0]).toBe(colors.get("Other")); + expect(series.colors[0]).not.toBe(OTHER_COLOR); + expect(series.labels.at(-1)).toBe("Other"); + expect(series.colors.at(-1)).toBe(OTHER_COLOR); + expect(series.labels).not.toContain("h"); + expect(colorForModel("h", colors)).toBe(client.colors[0]); + expect(bucketTotals(series).get("2026-10-01")).toBe(55); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts index 5d5386f8a3f..c103c2d7c60 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/overviewData.ts @@ -40,6 +40,20 @@ export const rollUpBreakdown = (results: readonly DailyData[], dimension: Breakd return [...totals.values()].sort((a, b) => b.spend - a.spend); }; +export const modelColorMap = (results: readonly DailyData[]): ReadonlyMap => { + const groups = rollUpBreakdown(results, "model_groups"); + const models = rollUpBreakdown(results, "models"); + const names = [...new Set([...groups, ...models].map((row) => row.key))]; + return new Map(names.map((name, index) => [name, stackedUsageColor(index)])); +}; + +export const colorForModel = (model: string, colorByModel: ReadonlyMap): string => { + const assignedColor = colorByModel.get(model); + if (assignedColor) return assignedColor; + const hash = Array.from(model).reduce((value, character) => (value * 31 + character.charCodeAt(0)) >>> 0, 0); + return stackedUsageColor(hash); +}; + export const OTHER_COLOR = "#94a3b8"; const OTHER_SERIES = "Other"; @@ -93,8 +107,9 @@ export const seriesBy = ( results: readonly DailyData[], dimension: BreakdownDimension, metric: UsageMetric, - top = 6, + options: number | { top?: number; colorByKey?: ReadonlyMap } = 6, ): Series => { + const { top = 6, colorByKey } = typeof options === "number" ? { top: options } : options; const leaders = rollUpBreakdown(results, dimension) .filter((row) => rankValue(row, metric) > 0) .sort((a, b) => rankValue(b, metric) - rankValue(a, metric)) @@ -118,9 +133,10 @@ export const seriesBy = ( label: formatDayLabel(day.date), ...Object.fromEntries((hasOther ? [...values, other] : values).map((value, index) => [keys[index], value])), })); - const colors = labels.map((label, index) => - hasOther && index === labels.length - 1 ? OTHER_COLOR : stackedUsageColor(index), - ); + const colors = labels.map((label, index) => { + if (hasOther && index === labels.length - 1) return OTHER_COLOR; + return colorByKey ? colorForModel(label, colorByKey) : stackedUsageColor(index); + }); return { data, keys, labels, colors }; }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/tokenBreakdown.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/tokenBreakdown.test.ts new file mode 100644 index 00000000000..dc5ccd30fc0 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/tokenBreakdown.test.ts @@ -0,0 +1,68 @@ +import { describe, expect, it } from "vitest"; +import { reconciledTokenBreakdown, type TokenBreakdown } from "./tokenBreakdown"; + +const recorded: TokenBreakdown = { + inputTokens: 80, + outputTokens: 20, + cacheReadTokens: 30, + cacheWriteTokens: 10, +}; +const unavailableCache = { ...recorded, cacheReadTokens: null, cacheWriteTokens: null }; + +describe("reconciledTokenBreakdown", () => { + it("should preserve recorded counts when input, output and cache reconcile", () => { + expect(reconciledTokenBreakdown(100, recorded)).toEqual(recorded); + }); + + it("should preserve known input and output but suppress cache when totals disagree", () => { + expect(reconciledTokenBreakdown(150, recorded)).toEqual(unavailableCache); + }); + + it("should suppress overlapping cache counts that exceed input tokens", () => { + const overlapping = { ...recorded, cacheReadTokens: 75 }; + expect(reconciledTokenBreakdown(100, overlapping)).toEqual(unavailableCache); + }); + + it("should keep recorded zero counts when the total is also zero", () => { + const zero = { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheWriteTokens: 0 }; + expect(reconciledTokenBreakdown(0, zero)).toEqual(zero); + }); + + it("should not fill missing cache counts with zero", () => { + const missingCache = { ...recorded, cacheReadTokens: null }; + expect(reconciledTokenBreakdown(100, missingCache)).toEqual(unavailableCache); + }); + + it.each([null, Number.NaN, Number.POSITIVE_INFINITY, -1])( + "should preserve known output but suppress cache when input is invalid (%s)", + (inputTokens) => { + const invalidInput = { ...recorded, inputTokens }; + const expected = { ...unavailableCache, inputTokens: null }; + expect(reconciledTokenBreakdown(100, invalidInput)).toEqual(expected); + }, + ); + + it.each([null, Number.NaN, Number.POSITIVE_INFINITY, -1])( + "should preserve known input but suppress cache when output is invalid (%s)", + (outputTokens) => { + const invalidOutput = { ...recorded, outputTokens }; + const expected = { ...unavailableCache, outputTokens: null }; + expect(reconciledTokenBreakdown(100, invalidOutput)).toEqual(expected); + }, + ); + + it.each([Number.NaN, Number.POSITIVE_INFINITY, -1])( + "should suppress invalid cache counts (%s)", + (cacheWriteTokens) => { + const invalidCache = { ...recorded, cacheWriteTokens }; + expect(reconciledTokenBreakdown(100, invalidCache)).toEqual(unavailableCache); + }, + ); + + it.each([Number.NaN, Number.POSITIVE_INFINITY, -1])( + "should suppress cache when the total is invalid (%s)", + (total) => { + expect(reconciledTokenBreakdown(total, recorded)).toEqual(unavailableCache); + }, + ); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/tokenBreakdown.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/tokenBreakdown.ts new file mode 100644 index 00000000000..6aa8eb797eb --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/overview/tokenBreakdown.ts @@ -0,0 +1,27 @@ +export interface TokenBreakdown { + inputTokens: number | null; + outputTokens: number | null; + cacheReadTokens: number | null; + cacheWriteTokens: number | null; +} + +const knownCount = (value: number | null): number | null => + value !== null && Number.isFinite(value) && value >= 0 ? value : null; + +export const reconciledTokenBreakdown = (total: number, breakdown: TokenBreakdown): TokenBreakdown => { + const inputTokens = knownCount(breakdown.inputTokens); + const outputTokens = knownCount(breakdown.outputTokens); + const cacheReadTokens = knownCount(breakdown.cacheReadTokens); + const cacheWriteTokens = knownCount(breakdown.cacheWriteTokens); + const inputReconciles = + inputTokens !== null && outputTokens !== null && inputTokens + outputTokens === knownCount(total); + const cacheCountsKnown = cacheReadTokens !== null && cacheWriteTokens !== null; + const cacheReconciles = inputReconciles && cacheCountsKnown && cacheReadTokens + cacheWriteTokens <= inputTokens; + + return { + inputTokens, + outputTokens, + cacheReadTokens: cacheReconciles ? cacheReadTokens : null, + cacheWriteTokens: cacheReconciles ? cacheWriteTokens : null, + }; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientAgentModelsData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientAgentModelsData.test.ts new file mode 100644 index 00000000000..065bc2f8d01 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientAgentModelsData.test.ts @@ -0,0 +1,111 @@ +import { describe, expect, it } from "vitest"; +import type { AgentRow } from "../components/overview/agentCatalog"; +import { clientAgentDetail } from "../components/overview/clientAgentDetail"; +import { clientAgentModelsFromExport } from "./clientAgentModelsData"; + +const desktop = "User-Agent: claude-cli/2.1.1 (external, claude-desktop-3p)"; +const code = "User-Agent: claude-cli/2.1.1 (external, cli)"; +const record = { + date: "2026-10-22", + entity_id: desktop, + model: "model-a", + spend: 1, + flat_cost: 0, + prompt_tokens: 80, + completion_tokens: 20, + api_requests: 2, + successful_requests: 2, + failed_requests: 0, + cache_read_input_tokens: 30, + cache_creation_input_tokens: 10, +}; +const agent: AgentRow = { + id: "claude-desktop", + label: "Claude Desktop", + description: "Desktop client", + kind: "agent", + logo: null, + spend: 3, + tokens: 300, + requests: 6, + users: 1, + tags: [desktop, desktop], +}; +const records = [ + { ...record, model: "model-b", spend: 2, prompt_tokens: 160, completion_tokens: 40, api_requests: 4 }, + record, + { ...record, entity_id: code, spend: 400, model: "code-only" }, + { ...record, entity_id: "User-Agent: claude-cli", spend: 500, model: "bare-rollup" }, +]; + +describe("clientAgentModelsFromExport", () => { + it("should return exact Desktop models without Code or bare-tag totals", () => { + const result = clientAgentModelsFromExport(records, agent); + + expect(result.error).toBeNull(); + expect(result.results).toHaveLength(1); + expect(Object.keys(result.results[0].breakdown.models)).toEqual(["model-a", "model-b"]); + expect(result.results[0].metrics).toMatchObject({ spend: 3, total_tokens: 300, api_requests: 6 }); + expect(result.results[0].breakdown.models["model-b"].metrics.spend).toBe(2); + expect(Object.keys(result.results[0].breakdown.entities)).toEqual([desktop]); + expect(clientAgentDetail(agent, result.results).dailyMatchesSummary).toBe(true); + }); + + it("should preserve recorded input and cache counts for the panel explanation", () => { + const { results } = clientAgentModelsFromExport(records, agent); + const expected = { inputTokens: 240, outputTokens: 60, cacheReadTokens: 60, cacheWriteTokens: 20 }; + expect(clientAgentDetail(agent, results).tokenBreakdown).toEqual(expected); + }); + + it("should sum the same model across canonical tags and sort days", () => { + const secondTag = "User-Agent: claude-desktop/3.0"; + const multiTagAgent = { ...agent, tags: [desktop, secondTag] }; + const rows = [record, { ...record, entity_id: secondTag }, { ...record, date: "2026-10-21" }]; + const { results } = clientAgentModelsFromExport(rows, multiTagAgent); + + expect(results.map((day) => day.date)).toEqual(["2026-10-21", "2026-10-22"]); + expect(results[1].breakdown.models["model-a"].metrics.total_tokens).toBe(200); + expect(results[1].metrics.spend).toBe(2); + }); + + it("should label missing model names without losing their recorded usage", () => { + const oneRow = { ...record, model: null }; + const singleAgent = { ...agent, spend: 1, tokens: 100, requests: 2 }; + const result = clientAgentModelsFromExport([oneRow], singleAgent); + + expect(result.error).toBeNull(); + expect(result.results[0].breakdown.models["Unknown model"].metrics.spend).toBe(1); + }); + + it.each(["cache_read_input_tokens", "prompt_tokens", "api_requests"])( + "should reject missing %s without inventing zero telemetry", + (key) => { + const incomplete = { ...record, [key]: undefined }; + const result = clientAgentModelsFromExport([incomplete], agent); + expect(result.error).toContain("incomplete"); + expect(result.results).toEqual([]); + }, + ); + + it("should reject duplicate tag, model and date rows rather than double-count them", () => { + const result = clientAgentModelsFromExport([...records, record], agent); + expect(result.error).toContain("duplicate"); + expect(result.results).toEqual([]); + }); + + it.each([{ spend: 4 }, { tokens: 400 }, { requests: 8 }])( + "should withhold models when the client summary changed (%o)", + (changed) => { + const result = clientAgentModelsFromExport(records, { ...agent, ...changed }); + expect(result.error).toContain("does not match"); + expect(result.results).toEqual([]); + }, + ); + + it("should never fall back to unrelated models when no exact tags match", () => { + const otherAgent = { ...agent, tags: ["User-Agent: absent"] }; + const result = clientAgentModelsFromExport(records, otherAgent); + expect(result.error).not.toBeNull(); + expect(result.results).toEqual([]); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientAgentModelsData.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientAgentModelsData.ts new file mode 100644 index 00000000000..61640a09de2 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientAgentModelsData.ts @@ -0,0 +1,90 @@ +import { z } from "zod"; +import type { DailyData, MetricWithMetadata, SpendMetrics } from "@/components/UsagePage/types"; +import type { AgentRow } from "../components/overview/agentCatalog"; +import { clientAgentDetail } from "../components/overview/clientAgentDetail"; + +const count = z.number().finite().nonnegative(); +const exportRowFields = { + date: z.string().regex(/^\d{4}-\d{2}-\d{2}$/), + entity_id: z.string(), + model: z.string().nullable(), + spend: z.number().finite(), + flat_cost: z.number().finite(), + prompt_tokens: count, + completion_tokens: count, + api_requests: count, + successful_requests: count, + failed_requests: count, + cache_read_input_tokens: count, + cache_creation_input_tokens: count, +}; +const exportRow = z.object(exportRowFields); +type ExportRow = z.infer; +type MetricKey = Exclude; + +export interface ClientAgentModelsResult { + results: DailyData[]; + error: string | null; +} + +const grouped = (rows: readonly ExportRow[], key: (row: ExportRow) => string) => { + const sorted = [...rows].sort((a, b) => key(a).localeCompare(key(b))); + const starts = sorted.flatMap((row, index) => (index === 0 || key(row) !== key(sorted[index - 1]) ? [index] : [])); + return starts.map((start, index) => [key(sorted[start]), sorted.slice(start, starts[index + 1])] as const); +}; + +const sum = (rows: readonly ExportRow[], key: MetricKey) => rows.reduce((total, row) => total + row[key], 0); + +const metricsFor = (rows: readonly ExportRow[]): SpendMetrics => ({ + spend: sum(rows, "spend"), + flat_cost: sum(rows, "flat_cost"), + prompt_tokens: sum(rows, "prompt_tokens"), + completion_tokens: sum(rows, "completion_tokens"), + total_tokens: sum(rows, "prompt_tokens") + sum(rows, "completion_tokens"), + api_requests: sum(rows, "api_requests"), + successful_requests: sum(rows, "successful_requests"), + failed_requests: sum(rows, "failed_requests"), + cache_read_input_tokens: sum(rows, "cache_read_input_tokens"), + cache_creation_input_tokens: sum(rows, "cache_creation_input_tokens"), +}); + +const breakdownFor = ( + rows: readonly ExportRow[], + key: (row: ExportRow) => string, +): Record => + Object.fromEntries( + grouped(rows, key).map(([name, members]) => [ + name, + { metrics: metricsFor(members), metadata: {}, api_key_breakdown: {} }, + ]), + ); + +export const clientAgentModelsFromExport = (payload: unknown, agent: AgentRow): ClientAgentModelsResult => { + const parsed = z.array(exportRow).safeParse(payload); + if (!parsed.success) return { results: [], error: "The proxy returned incomplete model usage data" }; + const tags = new Set(agent.tags); + const rows = parsed.data.filter((row) => tags.has(row.entity_id)); + const identities = rows.map((row) => JSON.stringify([row.date, row.entity_id, row.model])); + if (new Set(identities).size !== identities.length) { + return { results: [], error: "The proxy returned duplicate model usage data. Try again." }; + } + const results = grouped(rows, (row) => row.date).map(([date, members]) => ({ + date, + metrics: metricsFor(members), + breakdown: { + models: breakdownFor(members, (row) => row.model || "Unknown model"), + model_groups: {}, + entities: breakdownFor(members, (row) => row.entity_id), + providers: {}, + api_keys: {}, + mcp_servers: {}, + }, + })); + if (!clientAgentDetail(agent, results).dailyMatchesSummary) { + return { + results: [], + error: "Model usage does not match these client totals yet. Refresh or choose a shorter range.", + }; + } + return { results, error: null }; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.test.ts new file mode 100644 index 00000000000..2fa261e80c2 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from "vitest"; +import { + clientUsageFromLogs, + identifyUsageClient, + INCOMPLETE_CLIENT_USAGE, + UsageClient, + type ClientUsageScope, +} from "./clientUsageData"; + +const desktopAgent = "claude-cli/2.1.299 (external, claude-desktop-3p, agent-sdk/0.3.299)"; +const log = (overrides: Record = {}) => ({ + request_id: "request-1", + user: "user-1", + startTime: "2026-10-22T10:00:00Z", + model: "provider-model", + model_group: "deployment-alias", + custom_llm_provider: "anthropic", + spend: 0.013, + total_tokens: 1200, + prompt_tokens: 1000, + completion_tokens: 200, + request_duration_ms: 250, + status: "success", + request_tags: ["User-Agent: claude-cli", `User-Agent: ${desktopAgent}`], + metadata: { + user_agent: desktopAgent, + additional_usage_values: { cache_read_input_tokens: 500, cache_creation_input_tokens: 100 }, + }, + ...overrides, +}); +const page = (rows: ReturnType[], overrides: Record = {}) => ({ + data: rows, + total: rows.length, + page: 1, + total_pages: 1, + total_is_capped: false, + ...overrides, +}); +const scope: ClientUsageScope = { + userId: "user-1", + client: UsageClient.ClaudeDesktop, + startTime: new Date("2026-10-21T00:00:00Z"), + endTime: new Date("2026-10-23T00:00:00Z"), +}; + +describe("client usage from request logs", () => { + it.each([ + ["claude-desktop/1.0", "claude-desktop"], + ["Claude_Desktop/1.0", "claude-desktop"], + ["Claude Desktop/1.0", "claude-desktop"], + ["claude-code/1.0", "claude-code"], + ["claude_code/1.0", "claude-code"], + ["claude-user/1.0", "claude-code"], + ["claude-cli/1.0 (external, cli)", "claude-code"], + ["claude-cli/1.0 (external, claude-desktop-3p, cli)", "claude-desktop"], + ["claude-cli", null], + ["claude-cli/1.0 (external, agent-sdk/1.0)", null], + ])("should classify %s as %s consistently with the agent catalog", (agent, expected) => { + expect(identifyUsageClient(agent, [])).toBe(expected); + }); + + it("should preserve recorded totals, local days, model groups, providers and measured latency", () => { + const result = clientUsageFromLogs(page([log(), log({ request_id: "request-2", status: "failure" })]), scope); + expect(result.error).toBeNull(); + const expectedTotals = { + total_spend: 0.026, + total_tokens: 2400, + total_prompt_tokens: 2000, + total_completion_tokens: 400, + total_api_requests: 2, + total_successful_requests: 1, + total_failed_requests: 1, + total_cache_read_input_tokens: 1000, + total_cache_creation_input_tokens: 200, + total_response_time_ms: 250, + total_timed_requests: 1, + }; + expect(result.data.metadata).toMatchObject(expectedTotals); + expect(result.cacheAvailable).toBe(true); + const day = result.data.results?.[0]; + expect(day?.breakdown?.models?.["provider-model"].metrics?.api_requests).toBe(2); + expect(day?.breakdown?.model_groups?.["deployment-alias"].metrics?.spend).toBe(0.026); + expect(day?.breakdown?.providers?.anthropic.metrics?.total_tokens).toBe(2400); + expect(day?.breakdown?.api_keys).toEqual({}); + const timestamp = new Date("2026-10-22T10:00:00Z"); + const localDay = `${timestamp.getFullYear()}-${String(timestamp.getMonth() + 1).padStart(2, "0")}-${String(timestamp.getDate()).padStart(2, "0")}`; + expect(day?.date).toBe(localDay); + }); + + it("should deduplicate request ids without double counting bare and full user-agent tags", () => { + const result = clientUsageFromLogs(page([log(), log({ spend: 0.02 })]), scope); + expect(result.data.metadata?.total_api_requests).toBe(1); + expect(result.data.metadata?.total_spend).toBe(0.02); + expect(result.data.metadata?.total_tokens).toBe(1200); + }); + + it("should distinguish Desktop from explicit Code clients and exclude ambiguous CLI records", () => { + const rows = [ + log(), + log({ request_id: "code", request_tags: [], metadata: { user_agent: "claude-cli/2.1.299 (external, cli)" } }), + log({ request_id: "unknown", request_tags: ["User-Agent: claude-cli"], metadata: null }), + ]; + const desktop = clientUsageFromLogs(page(rows), scope); + const code = clientUsageFromLogs(page(rows), { ...scope, client: UsageClient.ClaudeCode }); + expect(desktop.data.metadata?.total_api_requests).toBe(1); + expect(code.data.metadata?.total_api_requests).toBe(1); + expect(identifyUsageClient("claude-code/3.0", [])).toBe("claude-code"); + expect(identifyUsageClient("claude-cli/3.0 (external, agent-sdk/0.3)", [])).toBeNull(); + expect(identifyUsageClient(null, JSON.stringify([`User-Agent: ${desktopAgent}`]))).toBe("claude-desktop"); + expect(identifyUsageClient(null, { "User-Agent": desktopAgent })).toBe("claude-desktop"); + expect(identifyUsageClient(null, [desktopAgent])).toBeNull(); + }); + + it.each([{ total: 1001, total_pages: 2 }, { total: 10000, total_is_capped: true }, { total_pages: 2 }])( + "should refuse totals for a partial page %j", + (overrides) => { + const result = clientUsageFromLogs(page([log()], overrides), scope); + expect(result.error).toBe(INCOMPLETE_CLIENT_USAGE); + expect(result.data.results).toEqual([]); + expect(result.data.metadata?.total_spend).toBe(0); + }, + ); + + it("should refuse malformed or unexpectedly truncated data instead of displaying inaccurate totals", () => { + expect(clientUsageFromLogs(page([log()], { total: 2 }), scope).error).toBeTruthy(); + expect(clientUsageFromLogs(page([log({ spend: "0.013" })]), scope).error).toBeTruthy(); + expect(clientUsageFromLogs(page([log({ startTime: "invalid" })]), scope).error).toBeTruthy(); + }); + + it("should reject rows attributed to another request user regardless of key metadata", () => { + const result = clientUsageFromLogs(page([log({ user: "someone-else" })]), scope); + expect(result.error).toBe("The proxy returned usage outside the selected user"); + expect(result.data.results).toEqual([]); + }); + + it("should keep unknown cache separate from recorded zero values", () => { + const unknown = clientUsageFromLogs(page([log({ status: null, metadata: { user_agent: desktopAgent } })]), scope); + expect(unknown.cacheAvailable).toBe(false); + expect(unknown.data.metadata?.total_tokens).toBe(1200); + expect(unknown.data.metadata?.total_spend).toBe(0.013); + const zero = clientUsageFromLogs( + page([ + log({ + metadata: { + user_agent: desktopAgent, + additional_usage_values: { cache_read_input_tokens: 0, cache_creation_input_tokens: 0 }, + }, + }), + ]), + scope, + ); + expect(zero.cacheAvailable).toBe(true); + expect(zero.data.metadata?.total_cache_read_input_tokens).toBe(0); + }); + + it("should use recorded nested cache counts and metadata status when direct fields are absent", () => { + const result = clientUsageFromLogs( + page([ + log({ + status: null, + metadata: { + user_agent: desktopAgent, + status: "failure", + usage_object: { prompt_tokens_details: { cached_tokens: 500, cache_creation_tokens: 100 } }, + }, + }), + ]), + scope, + ); + expect(result.cacheAvailable).toBe(true); + expect(result.data.metadata?.total_cache_read_input_tokens).toBe(500); + expect(result.data.metadata?.total_failed_requests).toBe(1); + expect(result.data.metadata?.total_timed_requests).toBe(0); + }); + + it("should enforce exact timestamp bounds after the API query rounds to seconds", () => { + const rows = [ + log({ request_id: "before", startTime: "2026-10-22T10:00:00.100Z" }), + log({ request_id: "inside", startTime: "2026-10-22T10:00:00.500" }), + log({ request_id: "after", startTime: "2026-10-22T10:00:01Z" }), + ]; + const result = clientUsageFromLogs(page(rows), { + ...scope, + startTime: new Date("2026-10-22T10:00:00.400Z"), + endTime: new Date("2026-10-22T10:00:00.800Z"), + }); + expect(result.data.metadata?.total_api_requests).toBe(1); + }); + + it.each([0, 100])("should retain legacy recorded cache_write_tokens of %s", (cacheWrite) => { + const result = clientUsageFromLogs( + page([ + log({ + metadata: { + user_agent: desktopAgent, + additional_usage_values: { prompt_tokens_details: { cached_tokens: 0, cache_write_tokens: cacheWrite } }, + }, + }), + ]), + scope, + ); + expect(result.cacheAvailable).toBe(true); + expect(result.data.metadata?.total_cache_creation_input_tokens).toBe(cacheWrite); + }); + + it("should allow a complete empty page or exactly 1,000 records", () => { + const empty = clientUsageFromLogs(page([]), scope); + expect(empty.error).toBeNull(); + expect(empty.data.results).toEqual([]); + const full = clientUsageFromLogs( + page(Array.from({ length: 1000 }, (_, index) => log({ request_id: `${index}` }))), + scope, + ); + expect(full.error).toBeNull(); + expect(full.data.metadata?.total_api_requests).toBe(1000); + }); + it("should aggregate all selected-team requests including unassigned team keys", () => { + const teamScope = { ...scope, userId: null, teamId: "engineering" }; + const rows = [log({ team_id: "engineering" }), log({ request_id: "team-key", user: null, team_id: "engineering" })]; + const result = clientUsageFromLogs(page(rows), teamScope); + expect(result.error).toBeNull(); + expect(result.data.metadata?.total_api_requests).toBe(2); + expect(result.data.metadata?.total_spend).toBe(0.026); + }); + + it.each([null, undefined, "different-team"])("should reject team results with mismatched team %s", (teamId) => { + const result = clientUsageFromLogs(page([log({ team_id: teamId })]), { ...scope, teamId: "engineering" }); + expect(result.error).toBe("The proxy returned usage outside the selected team"); + expect(result.data.results).toEqual([]); + }); + + it("should require both selected member and selected team to match", () => { + const memberScope = { ...scope, teamId: "engineering" }; + const ownOtherTeam = clientUsageFromLogs(page([log({ team_id: "design" })]), memberScope); + const otherTeamMember = clientUsageFromLogs(page([log({ team_id: "engineering", user: "user-2" })]), memberScope); + const member = clientUsageFromLogs(page([log({ team_id: "engineering" })]), memberScope); + expect(ownOtherTeam.error).toContain("outside the selected team"); + expect(otherTeamMember.error).toContain("outside the selected user"); + expect(member.error).toBeNull(); + expect(member.data.metadata?.total_api_requests).toBe(1); + }); + + it("should refuse unscoped request-log aggregation", () => { + expect(clientUsageFromLogs(page([log()]), { ...scope, userId: null }).error).toContain("Select a user or team"); + }); + + it("should exclude unrelated clients containing a Desktop marker from Desktop totals", () => { + const rows = [ + log(), + log({ + request_id: "unrelated-client", + request_tags: ["User-Agent: python-httpx/1.0 (claude-desktop-3p)"], + metadata: { user_agent: "python-httpx/1.0 (claude-desktop-3p)" }, + }), + log({ + request_id: "ambiguous-cli", + request_tags: ["User-Agent: claude-cli/1.0 (integration=claude-desktop-3p)"], + metadata: null, + }), + ]; + const result = clientUsageFromLogs(page(rows), scope); + expect(result.error).toBeNull(); + expect(result.data.metadata?.total_api_requests).toBe(1); + expect(result.data.metadata?.total_spend).toBe(0.013); + }); + + it.each([ + { requestTags: [" User Agent: claude-desktop/1.0 "] }, + { requestTags: { " User Agent ": "claude-desktop/1.0" } }, + { requestTags: { " User-Agent: claude-desktop/1.0 ": true } }, + ])("should recognize the catalog's normalized tag formats %j", ({ requestTags }) => { + const result = clientUsageFromLogs(page([log({ request_tags: requestTags, metadata: null })]), scope); + expect(result.data.metadata?.total_api_requests).toBe(1); + }); + + it("should retain recorded zero and positive cache counts from the Responses API usage shape", () => { + const responseUsage = { input_tokens_details: { cached_tokens: 0, cache_write_tokens: 100 } }; + const result = clientUsageFromLogs(page([log({ metadata: { usage_object: responseUsage } })]), scope); + expect(result.cacheAvailable).toBe(true); + expect(result.data.metadata?.total_cache_read_input_tokens).toBe(0); + expect(result.data.metadata?.total_cache_creation_input_tokens).toBe(100); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.ts new file mode 100644 index 00000000000..41e655d3e6e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageData.ts @@ -0,0 +1,234 @@ +import { z } from "zod"; +import { + EMPTY_DAILY_ACTIVITY_METADATA, + EMPTY_DAILY_ACTIVITY_RESPONSE, + type DailyActivityAggregatedResponse, +} from "@/components/UsagePage/dailyActivityApi"; +import type { components } from "@/lib/http/schema"; +import { extractPromptCacheTokens } from "@/utils/promptCacheUsage"; +import { isUserAgentTag, usageClientFromUserAgent, UsageClient } from "../components/overview/agentCatalog"; + +type DailyData = components["schemas"]["DailySpendData"]; +type MetricWithMetadata = components["schemas"]["MetricWithMetadata"]; +type SpendMetrics = components["schemas"]["SpendMetrics"]; + +export { UsageClient } from "../components/overview/agentCatalog"; +export const CLIENT_REQUEST_LIMIT = 1000; +export const INCOMPLETE_CLIENT_USAGE = + "Narrow the date range to fewer than 1,000 requests to see complete client usage"; + +const count = z.number().int().nonnegative(); +const cacheDetailsSchema = z.object({ + cached_tokens: count.nullish(), + cache_creation_tokens: count.nullish(), + cache_write_tokens: count.nullish(), +}); +const usageFields = { + cache_read_input_tokens: count.nullish(), + cache_creation_input_tokens: count.nullish(), + prompt_tokens_details: cacheDetailsSchema.nullish(), + input_tokens_details: cacheDetailsSchema.nullish(), +}; +const usageSchema = z.object(usageFields); +const metadataFields = { + user_agent: z.string().nullish(), + status: z.string().nullish(), + additional_usage_values: usageSchema.nullish(), + usage_object: usageSchema.nullish(), +}; +const logFields = { + request_id: z.string().min(1), + user: z.string().nullish(), + team_id: z.string().nullish(), + startTime: z + .string() + .transform((value) => { + const withZone = /(?:Z|[+-]\d{2}:\d{2})$/i.test(value) ? value : `${value}Z`; + return new Date(withZone); + }) + .refine((value) => Number.isFinite(value.getTime())), + model: z.string().nullish(), + model_group: z.string().nullish(), + custom_llm_provider: z.string().nullish(), + spend: z.number().finite(), + total_tokens: count, + prompt_tokens: count, + completion_tokens: count, + request_duration_ms: z.number().nonnegative().nullish(), + status: z.string().nullish(), + request_tags: z.unknown(), + metadata: z.object(metadataFields).nullish(), +}; +const logSchema = z.object(logFields); +const pageFields = { + data: z.array(logSchema), + total: count, + page: z.literal(1), + total_pages: count, + total_is_capped: z.boolean(), +}; +const pageSchema = z.object(pageFields); +type ClientLog = z.infer; + +export interface ClientUsageScope { + userId: string | null; + teamId?: string | null; + client: UsageClient; + startTime: Date; + endTime: Date; +} + +export interface ClientUsageResult { + data: DailyActivityAggregatedResponse; + error: string | null; + cacheAvailable: boolean; +} + +export const emptyClientUsage = (error: string | null = null): ClientUsageResult => ({ + data: EMPTY_DAILY_ACTIVITY_RESPONSE, + error, + cacheAvailable: false, +}); + +function userAgentTags(tags: unknown): string[] { + if (Array.isArray(tags)) return tags.filter((tag): tag is string => typeof tag === "string"); + if (typeof tags === "string") { + try { + return userAgentTags(JSON.parse(tags)); + } catch { + return [tags]; + } + } + if (tags !== null && typeof tags === "object") { + return Object.entries(tags).flatMap(([key, value]) => { + if (isUserAgentTag(key)) return [key]; + return isUserAgentTag(`${key.trim()}:`) && typeof value === "string" ? [`User-Agent: ${value}`] : []; + }); + } + return []; +} + +export function identifyUsageClient(metadataAgent: string | null | undefined, tags: unknown): UsageClient | null { + const candidates = [...(metadataAgent ? [metadataAgent] : []), ...userAgentTags(tags).filter(isUserAgentTag)].map( + usageClientFromUserAgent, + ); + if (candidates.includes(UsageClient.ClaudeDesktop)) return UsageClient.ClaudeDesktop; + if (candidates.includes(UsageClient.ClaudeCode)) return UsageClient.ClaudeCode; + return null; +} + +const statusOf = (row: ClientLog) => row.status ?? row.metadata?.status; +const cacheTokens = (row: ClientLog) => { + const additional = extractPromptCacheTokens(row.metadata?.additional_usage_values, true); + const usage = extractPromptCacheTokens(row.metadata?.usage_object, true); + return { + cacheReadTokens: additional.cacheReadTokens ?? usage.cacheReadTokens, + cacheCreationTokens: additional.cacheCreationTokens ?? usage.cacheCreationTokens, + }; +}; +const sum = (rows: readonly ClientLog[], value: (row: ClientLog) => number) => + rows.reduce((total, row) => total + value(row), 0); +const metricsFor = (rows: readonly ClientLog[]): SpendMetrics => ({ + flat_cost: 0, + ptu_hours: 0, + compression_saved_tokens: 0, + compression_savings_spend: 0, + prompt_caching_savings_spend: 0, + gateway_injected_caching_savings_spend: 0, + autorouter_savings_spend: 0, + spend: sum(rows, (row) => row.spend), + prompt_tokens: sum(rows, (row) => row.prompt_tokens), + completion_tokens: sum(rows, (row) => row.completion_tokens), + total_tokens: sum(rows, (row) => row.total_tokens), + api_requests: rows.length, + successful_requests: rows.filter((row) => statusOf(row) === "success").length, + failed_requests: rows.filter((row) => statusOf(row) === "failure").length, + cache_read_input_tokens: sum(rows, (row) => cacheTokens(row).cacheReadTokens ?? 0), + cache_creation_input_tokens: sum(rows, (row) => cacheTokens(row).cacheCreationTokens ?? 0), + total_response_time_ms: sum(rows, (row) => (statusOf(row) === "success" ? row.request_duration_ms ?? 0 : 0)), + timed_requests: rows.filter((row) => statusOf(row) === "success" && row.request_duration_ms != null).length, +}); +const breakdownFor = ( + rows: readonly ClientLog[], + dimension: (row: ClientLog) => string, +): Record => + Object.fromEntries( + [...new Set(rows.map(dimension))].map((name) => [ + name, + { + metrics: metricsFor(rows.filter((row) => dimension(row) === name)), + metadata: {}, + api_key_breakdown: {}, + }, + ]), + ); + +const localDay = (date: Date) => + `${date.getFullYear()}-${String(date.getMonth() + 1).padStart(2, "0")}-${String(date.getDate()).padStart(2, "0")}`; +const modelName = (row: ClientLog) => row.model || "Unknown model"; +const dailyData = (rows: readonly ClientLog[]): DailyData[] => + [...new Set(rows.map((row) => localDay(row.startTime)))].sort().map((date) => { + const day = rows.filter((row) => localDay(row.startTime) === date); + return { + date, + metrics: metricsFor(day), + breakdown: { + models: breakdownFor(day, modelName), + model_groups: breakdownFor(day, (row) => row.model_group || modelName(row)), + providers: breakdownFor(day, (row) => row.custom_llm_provider || "Unknown provider"), + api_keys: {}, + mcp_servers: {}, + entities: {}, + endpoints: {}, + }, + }; + }); + +export function clientUsageFromLogs(response: unknown, scope: ClientUsageScope): ClientUsageResult { + if (!scope.userId && !scope.teamId) return emptyClientUsage("Select a user or team to view client usage"); + const parsed = pageSchema.safeParse(response); + if (!parsed.success) return emptyClientUsage("The proxy returned incomplete client usage data"); + const page = parsed.data; + if (page.total_is_capped || page.total_pages > 1 || page.total > CLIENT_REQUEST_LIMIT) { + return emptyClientUsage(INCOMPLETE_CLIENT_USAGE); + } + if (page.data.length !== page.total) return emptyClientUsage("The proxy returned incomplete client usage data"); + if (scope.userId && page.data.some((row) => row.user !== scope.userId)) { + return emptyClientUsage("The proxy returned usage outside the selected user"); + } + if (scope.teamId && page.data.some((row) => row.team_id !== scope.teamId)) { + return emptyClientUsage("The proxy returned usage outside the selected team"); + } + const uniqueRows = [...new Map(page.data.map((row) => [row.request_id, row])).values()]; + const rows = uniqueRows.filter( + (row) => + row.startTime >= scope.startTime && + row.startTime <= scope.endTime && + identifyUsageClient(row.metadata?.user_agent, row.request_tags) === scope.client, + ); + const metrics = metricsFor(rows); + return { + data: { + results: dailyData(rows), + metadata: { + ...EMPTY_DAILY_ACTIVITY_METADATA, + total_spend: metrics.spend, + total_tokens: metrics.total_tokens, + total_prompt_tokens: metrics.prompt_tokens, + total_completion_tokens: metrics.completion_tokens, + total_api_requests: metrics.api_requests, + total_successful_requests: metrics.successful_requests, + total_failed_requests: metrics.failed_requests, + total_cache_read_input_tokens: metrics.cache_read_input_tokens, + total_cache_creation_input_tokens: metrics.cache_creation_input_tokens, + total_response_time_ms: metrics.total_response_time_ms, + total_timed_requests: metrics.timed_requests, + }, + }, + error: null, + cacheAvailable: rows.every((row) => { + const cache = cacheTokens(row); + return cache.cacheReadTokens !== undefined && cache.cacheCreationTokens !== undefined; + }), + }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageScope.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageScope.test.ts new file mode 100644 index 00000000000..6232f96167d --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageScope.test.ts @@ -0,0 +1,165 @@ +import { describe, expect, it } from "vitest"; +import { resolveClientUsageScope, type ClientUsageScopeInput, type ClientUsageTeam } from "./clientUsageScope"; + +const teams: readonly ClientUsageTeam[] = [ + { + team_id: "engineering", + team_alias: "Engineering", + members_with_roles: [ + { user_id: "john", role: "admin", user_alias: "John", user_email: "john@example.com" }, + { user_id: "alice", role: "user", user_email: "alice@example.com" }, + { user_id: "alice", role: "user", user_email: "duplicate@example.com" }, + { user_id: "id-only", role: "user" }, + { user_id: null, role: "user", user_email: "invited@example.com" }, + ], + }, + { + team_id: "design", + team_alias: "Design", + members_with_roles: [ + { user_id: "john", role: "admin" }, + { user_id: "bob", role: "user", user_alias: "Bob" }, + ], + }, + { + team_id: "external", + members_with_roles: [ + { user_id: "john", role: "user" }, + { user_id: "outside-admin", role: "admin" }, + ], + }, +]; + +const defaults: ClientUsageScopeInput = { + userId: "john", + userRole: "Internal User", + teams, + requestedTeamId: null, + requestedUserId: null, +}; + +const scopeFor = (overrides: Partial = {}) => + resolveClientUsageScope({ ...defaults, ...overrides }); + +describe("resolveClientUsageScope", () => { + it("should default a signed-in user to personal usage", () => { + const result = scopeFor({ userId: "alice" }); + expect(result.scope).toEqual({ userId: "alice", teamId: null }); + expect(result.canChooseGlobalUser).toBe(false); + expect(result.administeredTeams).toEqual([]); + expect(result.members).toEqual([]); + expect(result.error).toBeNull(); + }); + + it.each(["Admin", "Admin Viewer", "proxy_admin", "proxy_admin_viewer"])( + "should permit proxy-wide and selected-user read scopes for %s", + (userRole) => { + const allUsers = scopeFor({ userRole }); + const selectedUser = scopeFor({ userRole, requestedUserId: "any-proxy-user" }); + expect(allUsers.scope).toEqual({ userId: null, teamId: null }); + expect(allUsers.canChooseGlobalUser).toBe(true); + expect(allUsers.administeredTeams.map((team) => team.value)).toEqual(["engineering", "design", "external"]); + expect(selectedUser.scope).toEqual({ userId: "any-proxy-user", teamId: null }); + }, + ); + + it.each(["Org Admin", "org_admin", "Internal User", "internal_user", "team_admin"])( + "should not grant proxy-wide reads from the %s role name", + (userRole) => { + const result = scopeFor({ userRole, teams: [], requestedUserId: "outside-user" }); + expect(result.canChooseGlobalUser).toBe(false); + expect(result.scope).toBeNull(); + expect(result.error).toBe("Choose an administered team before viewing another member's usage"); + }, + ); + + it("should offer only administered teams and keep a team administrator in personal scope until a team is selected", () => { + const result = scopeFor(); + expect(result.scope).toEqual({ userId: "john", teamId: null }); + expect(result.administeredTeams).toEqual([ + { value: "engineering", label: "Engineering" }, + { value: "design", label: "Design" }, + ]); + expect(result.canChooseGlobalUser).toBe(false); + }); + + it("should expose only identified members of the selected administered team with their actual labels", () => { + const result = scopeFor({ requestedTeamId: "engineering" }); + expect(result.scope).toEqual({ teamId: "engineering", userId: null }); + expect(result.members).toEqual([ + { value: "john", label: "John (john@example.com)" }, + { value: "alice", label: "alice@example.com" }, + { value: "id-only", label: "id-only" }, + ]); + expect(result.error).toBeNull(); + }); + + it("should keep the team filter when selecting a team member or the administrator's own team usage", () => { + const alice = scopeFor({ requestedTeamId: "engineering", requestedUserId: "alice" }); + const john = scopeFor({ requestedTeamId: "engineering", requestedUserId: "john" }); + expect(alice.scope).toEqual({ teamId: "engineering", userId: "alice" }); + expect(john.scope).toEqual({ teamId: "engineering", userId: "john" }); + }); + + it("should reject a member from another administered team instead of querying that person's global usage", () => { + const result = scopeFor({ requestedTeamId: "engineering", requestedUserId: "bob" }); + expect(result.scope).toBeNull(); + expect(result.error).toBe("Choose a member of the selected team"); + expect(result.members.map((member) => member.value)).not.toContain("bob"); + }); + + it.each(["external", "missing-team"])("should reject the unauthorized team %s", (requestedTeamId) => { + const result = scopeFor({ requestedTeamId }); + expect(result.scope).toBeNull(); + expect(result.members).toEqual([]); + expect(result.error).toBe("Choose a team you administer"); + }); + + it("should reject a formerly valid selection as soon as team membership is removed", () => { + const result = scopeFor({ teams: [], requestedTeamId: "engineering", requestedUserId: "alice" }); + expect(result.scope).toBeNull(); + expect(result.administeredTeams).toEqual([]); + expect(result.members).toEqual([]); + }); + + it("should not use a missing team list to authorize a saved team or member selection", () => { + const result = scopeFor({ teams: undefined, requestedTeamId: "engineering", requestedUserId: "alice" }); + expect(result.scope).toBeNull(); + expect(result.error).toBe("Choose a team you administer"); + }); + + it("should restrict proxy-admin member options to the selected team too", () => { + const selected = { userRole: "Admin", requestedTeamId: "external" }; + const allowed = scopeFor({ ...selected, requestedUserId: "outside-admin" }); + const denied = scopeFor({ ...selected, requestedUserId: "alice" }); + expect(allowed.scope).toEqual({ teamId: "external", userId: "outside-admin" }); + expect(allowed.canChooseGlobalUser).toBe(false); + expect(denied.scope).toBeNull(); + }); + + it("should force personal usage for every role when ownUsage is set", () => { + const requested = { ownUsage: true, requestedTeamId: "missing-team", requestedUserId: "outside-user" }; + for (const userRole of ["Admin", "Internal User"]) { + const result = scopeFor({ ...requested, userRole }); + expect(result.scope).toEqual({ userId: "john", teamId: null }); + expect(result.members).toEqual([]); + expect(result.canChooseGlobalUser).toBe(false); + expect(result.error).toBeNull(); + } + }); + + it("should fail closed without a signed-in identity even if a proxy role or null-ID membership is present", () => { + const result = scopeFor({ userId: null, userRole: "Admin", requestedTeamId: "engineering" }); + expect(result.scope).toBeNull(); + expect(result.administeredTeams).toEqual([]); + expect(result.members).toEqual([]); + expect(result.canChooseGlobalUser).toBe(false); + expect(result.error).toBe("Sign in to view client usage"); + }); + + it("should not mutate the supplied team membership data", () => { + const before = JSON.stringify(teams); + scopeFor({ requestedTeamId: "engineering", requestedUserId: "alice" }); + expect(JSON.stringify(teams)).toBe(before); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageScope.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageScope.ts new file mode 100644 index 00000000000..702365fc22b --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/clientUsageScope.ts @@ -0,0 +1,98 @@ +import type { Member } from "@/components/networking"; +import { hasProxyWideSpendView, isUserTeamAdminForSingleTeam } from "@/utils/roles"; + +export interface ClientUsageTeam { + readonly team_id: string; + readonly team_alias?: string | null; + readonly members_with_roles?: readonly Member[] | null; +} + +export interface ClientUsageScopeOption { + value: string; + label: string; +} + +export interface ClientUsageScopeInput { + userId: string | null; + userRole: string | null; + teams: readonly ClientUsageTeam[] | null | undefined; + requestedTeamId: string | null; + requestedUserId: string | null; + ownUsage?: boolean; +} + +export interface ClientUsageScope { + userId: string | null; + teamId: string | null; +} + +export interface ResolvedClientUsageScope { + scope: ClientUsageScope | null; + administeredTeams: readonly ClientUsageScopeOption[]; + members: readonly ClientUsageScopeOption[]; + canChooseGlobalUser: boolean; + error: string | null; +} + +const memberLabel = (member: Member): string => { + const name = member.user_alias?.trim(); + const email = member.user_email?.trim(); + if (name && email && name !== email) return `${name} (${email})`; + if (name) return name; + return email || member.user_id || ""; +}; + +const memberOptions = (team: ClientUsageTeam): ClientUsageScopeOption[] => { + const members = team.members_with_roles ?? []; + const identified = members.filter((member): member is Member & { user_id: string } => Boolean(member.user_id)); + const unique = identified.filter( + (member, index) => identified.findIndex((row) => row.user_id === member.user_id) === index, + ); + return unique.map((member) => ({ value: member.user_id, label: memberLabel(member) })); +}; + +export const resolveClientUsageScope = ({ + userId, + userRole, + teams, + requestedTeamId, + requestedUserId, + ownUsage = false, +}: ClientUsageScopeInput): ResolvedClientUsageScope => { + const proxyAdmin = Boolean(userId) && hasProxyWideSpendView(userRole); + const availableTeams = userId + ? (teams ?? []).filter( + (team) => proxyAdmin || isUserTeamAdminForSingleTeam(team.members_with_roles ?? null, userId), + ) + : []; + const administeredTeams = availableTeams.map((team) => ({ + value: team.team_id, + label: team.team_alias || team.team_id, + })); + const base: ResolvedClientUsageScope = { + scope: null, + administeredTeams, + members: [], + canChooseGlobalUser: proxyAdmin && !ownUsage && !requestedTeamId, + error: null, + }; + if (!userId) return { ...base, error: "Sign in to view client usage" }; + if (ownUsage) return { ...base, scope: { userId, teamId: null } }; + + if (requestedTeamId) { + const selectedTeam = availableTeams.find((team) => team.team_id === requestedTeamId); + if (!selectedTeam) return { ...base, error: "Choose a team you administer" }; + const members = memberOptions(selectedTeam); + const memberId = requestedUserId || null; + if (memberId && !members.some((member) => member.value === memberId)) { + return { ...base, members, error: "Choose a member of the selected team" }; + } + return { ...base, members, scope: { teamId: selectedTeam.team_id, userId: memberId } }; + } + + if (proxyAdmin) return { ...base, scope: { teamId: null, userId: requestedUserId || null } }; + if (requestedUserId && requestedUserId !== userId) { + return { ...base, error: "Choose an administered team before viewing another member's usage" }; + } + return { ...base, scope: { teamId: null, userId } }; +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientAgentModels.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientAgentModels.test.ts new file mode 100644 index 00000000000..93298f87ca6 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientAgentModels.test.ts @@ -0,0 +1,168 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { dailyActivityExportCall } from "@/components/networking"; +import type { AgentRow } from "../components/overview/agentCatalog"; +import { MAX_CLIENT_MODEL_EXPORT_BYTES, useClientAgentModels } from "./useClientAgentModels"; + +vi.mock("@/components/networking", () => ({ dailyActivityExportCall: vi.fn() })); + +const tag = "User-Agent: claude-cli/2.1.1 (external, claude-desktop-3p)"; +const agent: AgentRow = { + id: "claude-desktop", + label: "Claude Desktop", + description: "Desktop client", + kind: "agent", + logo: null, + spend: 1, + tokens: 100, + requests: 2, + users: 1, + tags: [tag], +}; +const records = [ + { + date: "2026-10-22", + entity_id: tag, + model: "model-a", + spend: 1, + flat_cost: 0, + prompt_tokens: 80, + completion_tokens: 20, + api_requests: 2, + successful_requests: 2, + failed_requests: 0, + cache_read_input_tokens: 30, + cache_creation_input_tokens: 10, + }, +]; +const options = { + accessToken: "test-token", + agent, + enabled: true, + startTime: new Date("2026-10-22T07:00:00Z"), + endTime: new Date("2026-10-23T06:59:59.999Z"), +}; +const blob = (text: string) => Object.assign(new Blob([text]), { text: async () => text }); +const jsonBlob = (value: unknown) => blob(JSON.stringify(value)); + +describe("useClientAgentModels", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should request the complete authenticated model export for exact local filtering", async () => { + vi.mocked(dailyActivityExportCall).mockResolvedValue(jsonBlob(records)); + const { result } = renderHook(() => useClientAgentModels(options)); + await waitFor(() => expect(result.current.loading).toBe(false)); + const expectedRequest = { + accessToken: options.accessToken, + startTime: options.startTime, + endTime: options.endTime, + entityIds: null, + }; + + expect(dailyActivityExportCall).toHaveBeenCalledExactlyOnceWith( + "tag", + expectedRequest, + "daily_with_models", + "json", + ); + expect(result.current.error).toBeNull(); + expect(result.current.results[0].metrics.spend).toBe(1); + }); + + it("should require the caller's enabled gate and authentication", () => { + const { result, rerender } = renderHook( + ({ enabled, accessToken }) => useClientAgentModels({ ...options, enabled, accessToken }), + { initialProps: { enabled: false, accessToken: "test-token" as string | null } }, + ); + expect(result.current.results).toEqual([]); + rerender({ enabled: true, accessToken: null }); + expect(result.current.loading).toBe(false); + expect(dailyActivityExportCall).not.toHaveBeenCalled(); + }); + + it("should reject an oversized export without parsing or displaying partial totals", async () => { + const oversized = new Blob([new Uint8Array(MAX_CLIENT_MODEL_EXPORT_BYTES + 1)]); + vi.mocked(dailyActivityExportCall).mockResolvedValue(oversized); + const { result } = renderHook(() => useClientAgentModels(options)); + await waitFor(() => expect(result.current.error).toContain("shorter date range")); + expect(result.current.results).toEqual([]); + }); + + it("should reject incomplete JSON rather than consume part of an export", async () => { + vi.mocked(dailyActivityExportCall).mockResolvedValue(blob('[{"date":"2026-10-22"')); + const { result } = renderHook(() => useClientAgentModels(options)); + await waitFor(() => expect(result.current.error).toBe("Could not load complete model usage. Try again.")); + expect(result.current.results).toEqual([]); + }); + + it("should clear loaded results synchronously when the range changes", async () => { + vi.mocked(dailyActivityExportCall) + .mockResolvedValueOnce(jsonBlob(records)) + .mockReturnValueOnce(new Promise(() => {})); + const { result, rerender } = renderHook(({ startTime }) => useClientAgentModels({ ...options, startTime }), { + initialProps: { startTime: options.startTime }, + }); + await waitFor(() => expect(result.current.results).toHaveLength(1)); + rerender({ startTime: new Date("2026-10-21T07:00:00Z") }); + expect(result.current.results).toEqual([]); + expect(result.current.loading).toBe(true); + }); + + it("should ignore stale responses after the canonical tags change", async () => { + let resolveFirst: ((value: Blob) => void) | undefined; + const first = new Promise((resolve) => { + resolveFirst = resolve; + }); + const changedAgent = { ...agent, tags: ["User-Agent: claude-desktop/3.0"] }; + const changedRows = records.map((row) => ({ ...row, entity_id: changedAgent.tags[0] })); + vi.mocked(dailyActivityExportCall).mockReturnValueOnce(first).mockResolvedValueOnce(jsonBlob(changedRows)); + const { result, rerender } = renderHook(({ agent }) => useClientAgentModels({ ...options, agent }), { + initialProps: { agent }, + }); + rerender({ agent: changedAgent }); + await waitFor(() => expect(result.current.loading).toBe(false)); + await act(async () => resolveFirst?.(jsonBlob(records))); + expect(Object.keys(result.current.results[0].breakdown.entities)).toEqual(changedAgent.tags); + }); + + it("should clear loaded data immediately when the all-user panel is disabled", async () => { + vi.mocked(dailyActivityExportCall).mockResolvedValue(jsonBlob(records)); + const { result, rerender } = renderHook(({ enabled }) => useClientAgentModels({ ...options, enabled }), { + initialProps: { enabled: true }, + }); + await waitFor(() => expect(result.current.results).toHaveLength(1)); + rerender({ enabled: false }); + expect(result.current.results).toEqual([]); + expect(result.current.error).toBeNull(); + expect(result.current.loading).toBe(false); + }); + + it("should retry a failed export with unchanged scope and clear its error immediately", async () => { + vi.mocked(dailyActivityExportCall) + .mockRejectedValueOnce(new Error("Export unavailable")) + .mockResolvedValueOnce(jsonBlob(records)); + const { result } = renderHook(() => useClientAgentModels(options)); + await waitFor(() => expect(result.current.error).toBe("Export unavailable")); + + act(() => result.current.reload()); + expect(result.current.error).toBeNull(); + expect(result.current.loading).toBe(true); + await waitFor(() => expect(result.current.results).toHaveLength(1)); + expect(dailyActivityExportCall).toHaveBeenCalledTimes(2); + expect(vi.mocked(dailyActivityExportCall).mock.calls[1]).toEqual(vi.mocked(dailyActivityExportCall).mock.calls[0]); + }); + + it("should hide the previous model totals until a reload completes", async () => { + vi.mocked(dailyActivityExportCall) + .mockResolvedValueOnce(jsonBlob(records)) + .mockReturnValueOnce(new Promise(() => {})); + const { result } = renderHook(() => useClientAgentModels(options)); + await waitFor(() => expect(result.current.results).toHaveLength(1)); + + act(() => result.current.reload()); + expect(result.current.results).toEqual([]); + expect(result.current.loading).toBe(true); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientAgentModels.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientAgentModels.ts new file mode 100644 index 00000000000..9c97072c5ef --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientAgentModels.ts @@ -0,0 +1,67 @@ +import { useCallback, useEffect, useState } from "react"; +import { dailyActivityExportCall } from "@/components/networking"; +import type { DailyActivityRequest } from "@/components/UsagePage/dailyActivityApi"; +import type { AgentRow } from "../components/overview/agentCatalog"; +import { clientAgentModelsFromExport, type ClientAgentModelsResult } from "./clientAgentModelsData"; + +interface ClientAgentModelsOptions { + accessToken: string | null; + startTime: Date | null; + endTime: Date | null; + agent: AgentRow | null; + enabled: boolean; +} + +export const MAX_CLIENT_MODEL_EXPORT_BYTES = 20 * 1024 * 1024; + +const loadModels = async (request: DailyActivityRequest, agent: AgentRow): Promise => { + try { + const blob = await dailyActivityExportCall("tag", request, "daily_with_models", "json"); + if (blob.size > MAX_CLIENT_MODEL_EXPORT_BYTES) { + return { results: [], error: "Choose a shorter date range to load the model breakdown" }; + } + const payload: unknown = JSON.parse(await blob.text()); + return clientAgentModelsFromExport(payload, agent); + } catch (error: unknown) { + if (error instanceof SyntaxError) { + return { results: [], error: "Could not load complete model usage. Try again." }; + } + return { results: [], error: error instanceof Error ? error.message : "Could not load the model breakdown" }; + } +}; + +const agentKey = (agent: AgentRow | null) => + agent ? [agent.id, agent.tags, agent.spend, agent.tokens, agent.requests] : null; +const timestamp = (date: Date | null) => date?.getTime() ?? Number.NaN; + +export function useClientAgentModels({ accessToken, startTime, endTime, agent, enabled }: ClientAgentModelsOptions) { + const start = timestamp(startTime); + const end = timestamp(endTime); + const validRange = Number.isFinite(start) && Number.isFinite(end) && start <= end; + const active = enabled && Boolean(accessToken && agent && agent.tags.length > 0); + const [attempt, setAttempt] = useState(0); + const reload = useCallback(() => setAttempt((current) => current + 1), []); + const key = JSON.stringify([attempt, accessToken, start, end, agentKey(agent)]); + const [settled, setSettled] = useState<{ key: string; result: ClientAgentModelsResult } | null>(null); + + useEffect(() => { + if (!active || !validRange) return; + if (!accessToken || !agent) return; + let cancelled = false; + const request = { accessToken, startTime: new Date(start), endTime: new Date(end), entityIds: null }; + loadModels(request, agent).then((result) => { + if (!cancelled) setSettled({ key, result }); + }); + return () => { + cancelled = true; + }; + }, [active, validRange, accessToken, start, end, agent, key]); + + const current = active && validRange && settled?.key === key ? settled.result : null; + return { + results: current?.results ?? [], + loading: active && validRange && current === null, + error: active && !validRange ? "Select a valid date range" : current?.error ?? null, + reload, + }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientUsage.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientUsage.test.ts new file mode 100644 index 00000000000..8d93893c092 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientUsage.test.ts @@ -0,0 +1,169 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { uiSpendLogsCall } from "@/components/networking"; +import { UsageClient } from "./clientUsageData"; +import { useClientUsage } from "./useClientUsage"; + +vi.mock("@/components/networking", () => ({ uiSpendLogsCall: vi.fn() })); +const options = { + accessToken: "test-token", + userId: "user-1", + client: UsageClient.ClaudeDesktop, + startTime: new Date("2026-10-22T07:00:00.500Z"), + endTime: new Date("2026-10-23T06:59:59.999Z"), +}; +const emptyPage = { data: [], total: 0, page: 1, total_pages: 0, total_is_capped: false }; + +describe("useClientUsage", () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should send a bounded authenticated user query with UTC bounds covering the selected interval", async () => { + vi.mocked(uiSpendLogsCall).mockResolvedValue(emptyPage); + const { result } = renderHook(() => useClientUsage(options)); + await waitFor(() => expect(result.current.loading).toBe(false)); + const expectedRequest = { + accessToken: "test-token", + start_date: "2026-10-22 07:00:00", + end_date: "2026-10-23 07:00:00", + page: 1, + page_size: 1000, + params: { user_id: "user-1", sort_by: "startTime", sort_order: "desc", group_by_session: false }, + }; + expect(uiSpendLogsCall).toHaveBeenCalledExactlyOnceWith(expectedRequest); + expect(result.current.error).toBeNull(); + }); + + it("should require a selected user and avoid querying the all-clients view", () => { + const { result, rerender } = renderHook( + ({ userId, client }: { userId: string | null; client: UsageClient | null }) => + useClientUsage({ ...options, userId, client }), + { initialProps: { userId: null, client: UsageClient.ClaudeDesktop } }, + ); + expect(result.current.error).toBe("Select a user or team to view client usage"); + expect(result.current.loading).toBe(false); + rerender({ userId: "user-1", client: null }); + expect(result.current.error).toBeNull(); + expect(uiSpendLogsCall).not.toHaveBeenCalled(); + }); + + it("should expose incomplete results as an error without loading more pages", async () => { + vi.mocked(uiSpendLogsCall).mockResolvedValue({ ...emptyPage, total: 1001, total_pages: 2 }); + const { result } = renderHook(() => useClientUsage(options)); + await waitFor(() => expect(result.current.error).toContain("Narrow the date range")); + expect(result.current.data.results).toEqual([]); + expect(uiSpendLogsCall).toHaveBeenCalledTimes(1); + }); + + it("should expose network failures", async () => { + vi.mocked(uiSpendLogsCall).mockRejectedValue(new Error("No permission to view these logs")); + const { result } = renderHook(() => useClientUsage(options)); + await waitFor(() => expect(result.current.error).toBe("No permission to view these logs")); + expect(result.current.loading).toBe(false); + }); + + it("should ignore a stale response after the user changes", async () => { + let resolveFirst: ((value: unknown) => void) | undefined; + const first = new Promise((resolve) => { + resolveFirst = resolve; + }); + vi.mocked(uiSpendLogsCall).mockReturnValueOnce(first).mockResolvedValueOnce(emptyPage); + const { result, rerender } = renderHook(({ userId }) => useClientUsage({ ...options, userId }), { + initialProps: { userId: "user-1" }, + }); + rerender({ userId: "user-2" }); + await waitFor(() => expect(result.current.loading).toBe(false)); + await act(async () => { + resolveFirst?.({ ...emptyPage, total: 1001, total_pages: 2 }); + }); + expect(result.current.error).toBeNull(); + expect(result.current.data.metadata?.total_spend).toBe(0); + }); + + it("should clear loaded data immediately when client filtering is disabled", async () => { + vi.mocked(uiSpendLogsCall).mockResolvedValue({ ...emptyPage, total: 1001, total_pages: 2 }); + const { result, rerender } = renderHook( + ({ client }: { client: UsageClient | null }) => useClientUsage({ ...options, client }), + { initialProps: { client: UsageClient.ClaudeDesktop } }, + ); + await waitFor(() => expect(result.current.error).toContain("Narrow the date range")); + rerender({ client: null }); + expect(result.current.error).toBeNull(); + expect(result.current.loading).toBe(false); + expect(result.current.data.results).toEqual([]); + }); + + it("should retry a failed request with the same authenticated user and dates", async () => { + vi.mocked(uiSpendLogsCall).mockRejectedValueOnce(new Error("Logs unavailable")).mockResolvedValueOnce(emptyPage); + const { result } = renderHook(() => useClientUsage(options)); + await waitFor(() => expect(result.current.error).toBe("Logs unavailable")); + + act(() => result.current.reload()); + expect(result.current.error).toBeNull(); + expect(result.current.loading).toBe(true); + await waitFor(() => expect(result.current.loading).toBe(false)); + expect(uiSpendLogsCall).toHaveBeenCalledTimes(2); + expect(vi.mocked(uiSpendLogsCall).mock.calls[1]).toEqual(vi.mocked(uiSpendLogsCall).mock.calls[0]); + }); + + it("should hide previous person totals until a reload completes", async () => { + const row = { + request_id: "request-1", + user: options.userId, + startTime: "2026-10-22T12:00:00Z", + spend: 3, + total_tokens: 100, + prompt_tokens: 80, + completion_tokens: 20, + request_tags: [], + metadata: { user_agent: "claude-desktop/1.0" }, + }; + const page = { ...emptyPage, data: [row], total: 1, total_pages: 1 }; + vi.mocked(uiSpendLogsCall) + .mockResolvedValueOnce(page) + .mockReturnValueOnce(new Promise(() => {})); + const { result } = renderHook(() => useClientUsage(options)); + await waitFor(() => expect(result.current.data.metadata?.total_spend).toBe(3)); + + act(() => result.current.reload()); + expect(result.current.data.results).toEqual([]); + expect(result.current.loading).toBe(true); + }); + it("should query team totals and narrow to a member with both filters", async () => { + vi.mocked(uiSpendLogsCall).mockResolvedValue(emptyPage); + const initialProps: { userId: string | null } = { userId: null }; + const { result, rerender } = renderHook( + ({ userId }: { userId: string | null }) => useClientUsage({ ...options, userId, teamId: "engineering" }), + { initialProps }, + ); + await waitFor(() => expect(result.current.loading).toBe(false)); + const teamParams = { + team_id: "engineering", + sort_by: "startTime", + sort_order: "desc", + group_by_session: false, + }; + expect(vi.mocked(uiSpendLogsCall).mock.calls[0][0].params).toEqual(teamParams); + rerender({ userId: "user-1" }); + await waitFor(() => expect(result.current.loading).toBe(false)); + const memberParams = { ...teamParams, user_id: "user-1" }; + expect(vi.mocked(uiSpendLogsCall).mock.calls[1][0].params).toEqual(memberParams); + }); + + it("should ignore an old response when switching teams", async () => { + let resolveFirst: ((response: unknown) => void) | undefined; + const first = new Promise((resolve) => { + resolveFirst = resolve; + }); + vi.mocked(uiSpendLogsCall).mockReturnValueOnce(first).mockResolvedValueOnce(emptyPage); + const { result, rerender } = renderHook(({ teamId }) => useClientUsage({ ...options, teamId, userId: null }), { + initialProps: { teamId: "engineering" }, + }); + rerender({ teamId: "design" }); + await waitFor(() => expect(result.current.loading).toBe(false)); + await act(async () => resolveFirst?.({ ...emptyPage, total: 1001, total_pages: 2 })); + expect(result.current.error).toBeNull(); + expect(result.current.data.metadata?.total_spend).toBe(0); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientUsage.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientUsage.ts new file mode 100644 index 00000000000..42d30d46018 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useClientUsage.ts @@ -0,0 +1,77 @@ +import { useCallback, useEffect, useState } from "react"; +import { uiSpendLogsCall } from "@/components/networking"; +import { + CLIENT_REQUEST_LIMIT, + clientUsageFromLogs, + emptyClientUsage, + type ClientUsageResult, + type UsageClient, +} from "./clientUsageData"; + +interface ClientUsageOptions { + accessToken: string | null; + startTime: Date | null; + endTime: Date | null; + userId: string | null; + teamId?: string | null; + client: UsageClient | null; +} + +const utcSecond = (time: number) => new Date(time).toISOString().slice(0, 19).replace("T", " "); + +export function useClientUsage({ accessToken, startTime, endTime, userId, teamId = null, client }: ClientUsageOptions) { + const start = startTime?.getTime() ?? Number.NaN; + const end = endTime?.getTime() ?? Number.NaN; + const scopeError = (() => { + if (client === null) return null; + if (!userId && !teamId) return "Select a user or team to view client usage"; + if (!Number.isFinite(start) || !Number.isFinite(end) || start > end) return "Select a valid date range"; + if (!accessToken) return "Sign in to view client usage"; + return null; + })(); + const enabled = client !== null && scopeError === null; + const [attempt, setAttempt] = useState(0); + const reload = useCallback(() => setAttempt((current) => current + 1), []); + const key = JSON.stringify([attempt, accessToken, start, end, userId, teamId, client]); + const [settled, setSettled] = useState<{ key: string; result: ClientUsageResult } | null>(null); + + useEffect(() => { + if (!enabled) return; + if (!accessToken || !client) return; + if (!userId && !teamId) return; + let cancelled = false; + const scope = { userId, teamId, client, startTime: new Date(start), endTime: new Date(end) }; + const request = { + accessToken, + start_date: utcSecond(Math.floor(start / 1000) * 1000), + end_date: utcSecond(Math.ceil(end / 1000) * 1000), + page: 1, + page_size: CLIENT_REQUEST_LIMIT, + params: { + ...(userId ? { user_id: userId } : {}), + ...(teamId ? { team_id: teamId } : {}), + sort_by: "startTime", + sort_order: "desc" as const, + group_by_session: false, + }, + }; + uiSpendLogsCall(request) + .then((response: unknown) => { + if (!cancelled) setSettled({ key, result: clientUsageFromLogs(response, scope) }); + }) + .catch((error: unknown) => { + if (!cancelled) { + setSettled({ + key, + result: emptyClientUsage(error instanceof Error ? error.message : "Could not load client usage"), + }); + } + }); + return () => { + cancelled = true; + }; + }, [enabled, accessToken, start, end, userId, teamId, client, key]); + + const current = enabled && settled?.key === key ? settled.result : emptyClientUsage(scopeError); + return { ...current, loading: enabled && settled?.key !== key, reload }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useUsagePanelContext.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useUsagePanelContext.test.ts new file mode 100644 index 00000000000..24a07cb1281 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useUsagePanelContext.test.ts @@ -0,0 +1,85 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it } from "vitest"; +import { useUsagePanelContext } from "./useUsagePanelContext"; + +const dashboards: HTMLDivElement[] = []; + +function createDashboard() { + const viewport = document.createElement("div"); + const scroller = document.createElement("div"); + const page = document.createElement("div"); + const trigger = document.createElement("button"); + const panelControl = document.createElement("button"); + trigger.textContent = "View Claude Desktop usage"; + panelControl.textContent = "Back to Usage"; + page.append(trigger); + scroller.append(page); + viewport.append(scroller, panelControl); + document.body.append(viewport); + dashboards.push(viewport); + return { viewport, scroller, page, trigger, panelControl }; +} + +afterEach(() => { + dashboards.splice(0).forEach((dashboard) => dashboard.remove()); +}); + +describe("useUsagePanelContext", () => { + it("should preserve nested scroll positions while open and return focus to the trigger on close", async () => { + const { viewport, scroller, page, trigger, panelControl } = createDashboard(); + const { result, rerender } = renderHook(({ open }) => useUsagePanelContext(open), { + initialProps: { open: false }, + }); + act(() => { + result.current.pageRef.current = page; + viewport.scrollTop = 160; + scroller.scrollTop = 840; + scroller.scrollLeft = 45; + trigger.focus(); + result.current.remember(); + panelControl.focus(); + viewport.scrollTop = 0; + scroller.scrollTop = 0; + scroller.scrollLeft = 0; + }); + + rerender({ open: true }); + await waitFor(() => expect(scroller.scrollTop).toBe(840)); + expect(viewport.scrollTop).toBe(160); + expect(scroller.scrollLeft).toBe(45); + expect(panelControl).toHaveFocus(); + + scroller.scrollTop = 12; + viewport.scrollTop = 0; + rerender({ open: false }); + await waitFor(() => expect(trigger).toHaveFocus()); + expect(viewport.scrollTop).toBe(160); + expect(scroller.scrollTop).toBe(840); + expect(scroller.scrollLeft).toBe(45); + }); + + it("should restore the new position when a user reopens the panel after scrolling", async () => { + const { scroller, page, trigger, panelControl } = createDashboard(); + const { result, rerender } = renderHook(({ open }) => useUsagePanelContext(open), { + initialProps: { open: false }, + }); + act(() => { + result.current.pageRef.current = page; + }); + + for (const position of [250, 930]) { + act(() => { + scroller.scrollTop = position; + trigger.focus(); + result.current.remember(); + panelControl.focus(); + scroller.scrollTop = 0; + }); + rerender({ open: true }); + await waitFor(() => expect(scroller.scrollTop).toBe(position)); + rerender({ open: false }); + await waitFor(() => expect(trigger).toHaveFocus()); + expect(scroller.scrollTop).toBe(position); + } + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useUsagePanelContext.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useUsagePanelContext.ts new file mode 100644 index 00000000000..7db666ace3e --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/hooks/useUsagePanelContext.ts @@ -0,0 +1,37 @@ +import { useCallback, useLayoutEffect, useRef } from "react"; + +/** Keep the dashboard's nested scroll container in place across dialog focus changes. */ +export function useUsagePanelContext(open: boolean) { + const pageRef = useRef(null); + const snapshot = useRef<{ + scroll: { element: HTMLElement; top: number; left: number }[]; + focus: HTMLElement | null; + } | null>(null); + const remember = useCallback(() => { + const scroll: { element: HTMLElement; top: number; left: number }[] = []; + for (let element = pageRef.current?.parentElement; element; element = element.parentElement) { + scroll.push({ element, top: element.scrollTop, left: element.scrollLeft }); + } + const focus = document.activeElement instanceof HTMLElement ? document.activeElement : null; + snapshot.current = { scroll, focus }; + }, []); + + useLayoutEffect(() => { + const saved = snapshot.current; + if (!saved) return; + const restore = () => { + for (const { element, top, left } of saved.scroll) { + element.scrollTop = top; + element.scrollLeft = left; + } + }; + restore(); + const frame = requestAnimationFrame(() => { + if (!open) saved.focus?.focus({ preventScroll: true }); + restore(); + if (!open) snapshot.current = null; + }); + return () => cancelAnimationFrame(frame); + }, [open]); + return { pageRef, remember }; +} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/usageUrlState.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/usageUrlState.test.ts index dbb5c44cc35..1255ea03dd9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/usageUrlState.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/usageUrlState.test.ts @@ -2,10 +2,14 @@ import moment from "moment"; import { createLoader, createSerializer } from "nuqs"; import { describe, expect, it } from "vitest"; +import { UsageClient } from "./components/overview/agentCatalog"; import type { UsageOption } from "./components/UsageViewSelect/UsageViewSelect"; import { USAGE_OPTIONS } from "./components/UsageViewSelect/UsageViewSelect"; import { cleanUsageUrl, + clientDateRangeFromParams, + clientDateRangePatch, + clientPanelPatch, dateRangeFromParams, dateRangePatch, entitySelectionPatch, @@ -265,3 +269,105 @@ describe("cross-page links", () => { expect(params.user).toBe("alice+ops@example.com"); }); }); + +describe("client usage filters", () => { + it("should preserve a recognized client and person in a share link", () => { + const params = load("?user=u1&client=claude-desktop&range=7d"); + expect(params.client).toBe("claude-desktop"); + expect(clean("?user=u1&client=claude-desktop&range=7d")).toBeNull(); + expect(load(serialize(params)).user).toBe("u1"); + }); + + it("should reject an unknown client", () => { + expect(load("?client=made-up").client).toBeNull(); + }); + + it("should clear the client when changing to another usage view", () => { + expect(load(applyPatch("?client=claude-desktop&user=u1", usageViewPatch("team"))).client).toBeNull(); + expect(clean("?view=organization&client=claude-desktop")).toEqual({ + query: "?view=organization", + deniedAccess: false, + }); + expect(clean("?view=team&client=claude-desktop")).toBeNull(); + }); + + it("should preserve the underlying tab and person when a client panel opens", () => { + const query = applyPatch("?user=u1&tab=keys&range=7d", { client: UsageClient.ClaudeDesktop, client_user: "u2" }); + expect(usageTabFromParams(load(query))).toBe("keys"); + expect(clean(query)).toBeNull(); + expect(load(query).user).toBe("u1"); + expect(load(query).client_user).toBe("u2"); + expect(applyPatch(query, { client: null, client_user: null })).toBe("?user=u1&tab=keys&range=7d"); + }); + + it("should remove stray panel person filters", () => { + expect(clean("?client_user=u2&range=7d")).toEqual({ query: "?range=7d", deniedAccess: false }); + expect(clean("?client=claude-desktop&client_user=u2", internalUserAccess)).toEqual({ + query: "?client=claude-desktop", + deniedAccess: false, + }); + }); +}); + +describe("client panel date range", () => { + it("should change the panel dates without changing overview filters or dates", () => { + const query = applyPatch("?user=u1&tab=models&range=30d&client=claude-desktop", clientDateRangePatch({}, "7d")); + const params = load(query); + expect(params.range).toBe("30d"); + expect(params.user).toBe("u1"); + expect(params.tab).toBe("models"); + expect(params.client_range).toBe("7d"); + expect(clientDateRangeFromParams(params)).toBeDefined(); + expect(clean(query)).toBeNull(); + }); + + it("should round-trip custom panel dates independently", () => { + const value = { from: new Date(2026, 9, 20), to: new Date(2026, 9, 22) }; + const query = applyPatch("?client=claude-desktop&client_range=7d&range=30d", clientDateRangePatch(value, null)); + const params = load(query); + expect(params.client_range).toBeNull(); + expect(params.client_from).toBe("2026-10-20"); + expect(params.client_to).toBe("2026-10-22"); + expect(moment(clientDateRangeFromParams(params)?.from).format("YYYY-MM-DD")).toBe("2026-10-20"); + expect(params.range).toBe("30d"); + }); + + it("should clear only panel state on close or when opening a different client", () => { + const query = "?client=claude-desktop&client_user=u2&client_range=7d&range=30d&user=u1&tab=keys"; + expect(applyPatch(query, clientPanelPatch(null))).toBe("?range=30d&user=u1&tab=keys"); + const next = load(applyPatch(query, clientPanelPatch(UsageClient.ClaudeCode))); + expect(next.client).toBe("claude-code"); + expect(next.client_range).toBeNull(); + expect(next.range).toBe("30d"); + }); + + it("should remove invalid or orphaned panel dates without clearing overview dates", () => { + expect(clean("?range=7d&client_range=30d")).toEqual({ query: "?range=7d", deniedAccess: false }); + expect(clean("?range=7d&client=claude-desktop&client_from=bad&client_to=2026-10-22")).toEqual({ + query: "?range=7d&client=claude-desktop", + deniedAccess: false, + }); + }); +}); + +describe("client team scope", () => { + it("should preserve an administered-team member filter and independent dates", () => { + const query = + "?view=team&team=engineering&client=claude-desktop&client_team=engineering&client_user=member&client_range=30d"; + const access = { allowedViews: USAGE_OPTIONS, isAdmin: false, canViewTeamClients: true }; + expect(cleanUsageUrl(load(query), access)).toBeNull(); + }); + + it("should discard an unscoped person filter for an internal user", () => { + const cleanup = cleanUsageUrl(load("?client=claude-desktop&client_user=other"), internalUserAccess); + expect(cleanup?.patch).toMatchObject({ client_user: null }); + }); + + it("should clear panel team and member without changing the overview's selected teams", () => { + const query = applyPatch( + "?view=team&team=engineering&client=claude-desktop&client_team=design&client_user=member", + clientPanelPatch(null), + ); + expect(query).toBe("?view=team&team=engineering"); + }); +}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/usageUrlState.ts b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/usageUrlState.ts index a1cd04b723a..2af113412d9 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/usageUrlState.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/usageUrlState.ts @@ -1,11 +1,12 @@ import moment from "moment"; -import { parseAsNativeArrayOf, parseAsString, type inferParserType } from "nuqs"; +import { parseAsNativeArrayOf, parseAsString, parseAsStringEnum, type inferParserType } from "nuqs"; import type { EntityType } from "@/components/EntityUsageExport/types"; import { relativeTimeOptions } from "@/components/shared/advanced_date_picker"; import type { DateRangePickerValue } from "@/components/shared/date_picker_types"; import { uiHref } from "@/utils/uiHref"; import { USAGE_OPTIONS, type UsageOption } from "./components/UsageViewSelect/UsageViewSelect"; +import { UsageClient } from "./components/overview/agentCatalog"; export const DEFAULT_USAGE_VIEW: UsageOption = "global"; export const USAGE_TABS = ["cost", "models", "keys", "users", "mcp", "endpoints", "errors"] as const; @@ -21,6 +22,12 @@ export const usageUrlParsers = { from: parseAsString, to: parseAsString, user: parseAsString, + client_user: parseAsString, + client_team: parseAsString, + client_range: parseAsString, + client_from: parseAsString, + client_to: parseAsString, + client: parseAsStringEnum(Object.values(UsageClient)), team: entityIds, org: entityIds, customer: entityIds, @@ -54,12 +61,20 @@ const VIEW_ENTITY_KEYS: Readonly> = { "user-agent-activity": ["tag"], }; +export const CLIENT_USAGE_VIEWS: readonly UsageOption[] = ["global", "my-usage", "team"]; + const TABBED_VIEWS: readonly UsageOption[] = ["global", "my-usage"]; const RESET_TO_DEFAULT_VIEW: UsageUrlPatch = { view: null, tab: null, user: null, + client: null, + client_user: null, + client_team: null, + client_range: null, + client_from: null, + client_to: null, team: null, org: null, customer: null, @@ -77,8 +92,9 @@ const hasEntityValue = (params: UsageUrlParams, key: EntityKey): boolean => export const usageViewFromParams = (params: UsageUrlParams): UsageOption => isUsageOption(params.view) ? params.view : DEFAULT_USAGE_VIEW; -export const usageTabFromParams = (params: UsageUrlParams, isAdmin = true): UsageTab => - isUsageTab(params.tab) && (params.tab !== "errors" || isAdmin) ? params.tab : DEFAULT_USAGE_TAB; +export const usageTabFromParams = (params: UsageUrlParams, isAdmin = true): UsageTab => { + return isUsageTab(params.tab) && (params.tab !== "errors" || isAdmin) ? params.tab : DEFAULT_USAGE_TAB; +}; export const usageViewPatch = (view: UsageOption): UsageUrlPatch => ({ ...RESET_TO_DEFAULT_VIEW, @@ -128,6 +144,35 @@ export const dateRangePatch = (value: DateRangePickerValue, presetShortLabel: st }; }; +type ClientDateParams = Pick; + +const clientDateParams = (params: ClientDateParams): DateParams => ({ + range: params.client_range, + from: params.client_from, + to: params.client_to, +}); + +export const clientDateRangeFromParams = (params: ClientDateParams): DateRangePickerValue | undefined => + dateRangeFromParams(clientDateParams(params)); + +const asClientDatePatch = (patch: UsageUrlPatch): UsageUrlPatch => ({ + ...("range" in patch ? { client_range: patch.range } : {}), + ...("from" in patch ? { client_from: patch.from } : {}), + ...("to" in patch ? { client_to: patch.to } : {}), +}); + +export const clientDateRangePatch = (value: DateRangePickerValue, presetShortLabel: string | null): UsageUrlPatch => + asClientDatePatch(dateRangePatch(value, presetShortLabel)); + +export const clientPanelPatch = (client: UsageUrlParams["client"]): UsageUrlPatch => ({ + client, + client_user: null, + client_team: null, + client_range: null, + client_from: null, + client_to: null, +}); + const invalidDatePatch = (params: DateParams): UsageUrlPatch => { const hasCustomRange = params.from !== null || params.to !== null; if (params.range !== null) { @@ -144,6 +189,7 @@ const invalidDatePatch = (params: DateParams): UsageUrlPatch => { export interface UsageUrlAccess { readonly allowedViews: readonly UsageOption[]; readonly isAdmin: boolean; + readonly canViewTeamClients?: boolean; } export interface UsageUrlCleanup { @@ -166,9 +212,25 @@ export const cleanUsageUrl = (params: UsageUrlParams, access: UsageUrlAccess): U const invalidTab = params.tab !== null && (!TABBED_VIEWS.includes(view) || !isUsageTab(params.tab) || (!access.isAdmin && params.tab === "errors")); + const panelOpen = CLIENT_USAGE_VIEWS.includes(view) && params.client !== null; + const canFilterClientUser = access.isAdmin || Boolean(access.canViewTeamClients && params.client_team); + const panelDates = clientDateParams(params); + const panelDateCleanup = panelOpen + ? invalidDatePatch(panelDates) + : Object.fromEntries( + Object.entries(panelDates) + .filter(([, value]) => value !== null) + .map(([key]) => [key, null]), + ); const patch: UsageUrlPatch = { + ...asClientDatePatch(panelDateCleanup), ...(params.view !== null && !isUsageOption(params.view) ? { view: null } : {}), ...(invalidTab ? { tab: null } : {}), + ...(!CLIENT_USAGE_VIEWS.includes(view) && params.client !== null ? { client: null } : {}), + ...((!panelOpen || !canFilterClientUser) && params.client_user !== null ? { client_user: null } : {}), + ...((!CLIENT_USAGE_VIEWS.includes(view) || params.client === null) && params.client_team !== null + ? { client_team: null } + : {}), ...Object.fromEntries(strayEntityKeys.map((key) => [key, null])), ...invalidDatePatch(params), }; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx index 32ce0a69aea..a719131678f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/page.tsx @@ -7,7 +7,7 @@ import { useOrganizations } from "@/app/(dashboard)/hooks/organizations/useOrgan export default function UsagePage() { useAuthorized(); - const { data: teams } = useTeams(); + const { data: teams, isLoading: teamsLoading } = useTeams(); const { data: organizations } = useOrganizations(); - return ; + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.test.tsx index 9b7e60aa12a..ba3d1d0fb7d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/vector-stores/_components/CreateVectorStore.test.tsx @@ -270,15 +270,16 @@ describe("CreateVectorStore", () => { }); it("should display S3 Vectors provider-specific fields when selected", async () => { + const user = userEvent.setup(); render(); // Find and click the provider dropdown const providerSelect = screen.getByRole("combobox"); - await userEvent.click(providerSelect); + await user.click(providerSelect); // Wait for dropdown options to appear - await userEvent.click(await screen.findByText("AWS S3 Vectors")); + await user.click(await screen.findByRole("option", { name: /AWS S3 Vectors/ })); // Check if S3-specific fields are displayed await waitFor(() => { @@ -289,6 +290,7 @@ describe("CreateVectorStore", () => { }); it("should validate S3 Vectors required fields before submission", async () => { + const user = userEvent.setup(); render(); // Upload a file first @@ -306,11 +308,11 @@ describe("CreateVectorStore", () => { }); // Select S3 Vectors provider - const providerSelect = screen.getByRole("combobox"); + const providerSelect = screen.getByRole("combobox", { name: /Provider/ }); - await userEvent.click(providerSelect); + await user.click(providerSelect); - await userEvent.click(await screen.findByText("AWS S3 Vectors")); + await user.click(await screen.findByRole("option", { name: /AWS S3 Vectors/ })); // Try to create without filling required fields const createButton = screen.getByRole("button", { name: /Create Vector Store/i }); @@ -319,7 +321,7 @@ describe("CreateVectorStore", () => { fireEvent.click(createButton); }); - // Should show validation warning (mocked message.warning would be called) - // The actual validation happens in the component + expect(toast.warning).toHaveBeenCalledWith("Please provide Vector Bucket Name"); + expect(networking.ragIngestCall).not.toHaveBeenCalled(); }); }); diff --git a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx index 6da8e7c4a92..72d557d1f8f 100644 --- a/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/edit_auto_router/edit_auto_router_modal.integration.test.tsx @@ -90,6 +90,17 @@ const savedConfig = () => { return payload?.litellm_params?.complexity_router_config; }; +const renderWithSettledAvailability = async (renderModal: () => unknown) => { + vi.useFakeTimers(); + try { + renderModal(); + await act(() => vi.advanceTimersByTimeAsync(300)); + await act(() => vi.runOnlyPendingTimersAsync()); + } finally { + vi.useRealTimers(); + } +}; + describe("EditAutoRouterModal keyword matching", () => { beforeEach(() => { modelPatchUpdateCall.mockClear(); @@ -1447,7 +1458,7 @@ describe("EditAutoRouterModal prompt compression", () => { { auto_router_model_compression: "model-compressor" }, ])("should preserve the exact stored compression fields through an untouched save: %j", async (stored) => { const user = userEvent.setup(); - renderWithStoredCompression(stored); + await renderWithSettledAvailability(() => renderWithStoredCompression(stored)); await waitFor(() => expect(screen.getByRole("button", { name: /save changes/i })).toBeEnabled()); await user.click(await screen.findByRole("button", { name: /save changes/i })); @@ -1524,7 +1535,7 @@ describe("EditAutoRouterModal classifier vision", () => { it("hydrates and keeps a stored vision setting through an untouched save", async () => { const user = userEvent.setup(); - renderModal(); + await renderWithSettledAvailability(renderModal); openAutoRouterAdvanced("Classification Method"); expect(screen.getByRole("switch", { name: "Use images for classification" })).toBeChecked(); diff --git a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.test.tsx b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.test.tsx index 104eed54b2f..d722b6d1ad6 100644 --- a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.test.tsx +++ b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.test.tsx @@ -1,4 +1,4 @@ -import { render, screen, fireEvent, waitFor } from "@testing-library/react"; +import { act, render, screen, fireEvent, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import moment from "moment"; import { describe, it, expect, vi, beforeEach, beforeAll } from "vitest"; @@ -312,4 +312,48 @@ describe("AdvancedDatePicker", () => { expect(applyButton).toBeDisabled(); }); }); + + it("should label compact custom date inputs, validate their order and apply the corrected range", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { expanded: false })); + const start = screen.getByLabelText("Start date"); + const end = screen.getByLabelText("End date"); + expect(start).toHaveValue("2025-01-01"); + expect(end).toHaveValue("2025-01-31"); + + act(() => fireEvent.change(start, { target: { value: "2025-02-01" } })); + expect(screen.getByText("End date cannot be before start date")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Apply" })).toBeDisabled(); + expect(mockOnValueChange).not.toHaveBeenCalled(); + + act(() => fireEvent.change(end, { target: { value: "2025-02-05" } })); + expect(screen.queryByText("End date cannot be before start date")).not.toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Apply" })).toBeEnabled(); + expect(mockOnValueChange).not.toHaveBeenCalled(); + + await user.click(screen.getByRole("button", { name: "Apply" })); + const expectedRange = { + from: new Date(2025, 1, 1), + to: new Date(2025, 1, 5, 23, 59, 59, 999), + }; + expect(mockOnValueChange).toHaveBeenNthCalledWith(1, expectedRange, null); + await waitFor(() => expect(mockOnValueChange).toHaveBeenNthCalledWith(2, expectedRange, null)); + expect(screen.queryByLabelText("Start date")).not.toBeInTheDocument(); + }); + + it("should keep compact preset selection pending until Apply is clicked", async () => { + const user = userEvent.setup(); + render(); + + await user.click(screen.getByRole("button", { expanded: false })); + await user.click(screen.getByRole("button", { name: /Last 7 days/ })); + expect(screen.getByRole("button", { name: /Last 7 days/ })).toHaveAttribute("aria-pressed", "true"); + expect(mockOnValueChange).not.toHaveBeenCalled(); + + await user.click(screen.getByRole("button", { name: "Apply" })); + expect(mockOnValueChange).toHaveBeenNthCalledWith(1, expect.anything(), "7d"); + await waitFor(() => expect(mockOnValueChange).toHaveBeenNthCalledWith(2, expect.anything(), "7d")); + }); }); diff --git a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx index 513db07409c..27ed237720f 100644 --- a/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx +++ b/ui/litellm-dashboard/src/components/shared/advanced_date_picker.tsx @@ -3,7 +3,7 @@ import { Button } from "@/components/ui/button"; import { cn } from "@/lib/cva.config"; import type { DateRangePickerValue } from "./date_picker_types"; import moment from "moment"; -import React, { useCallback, useEffect, useRef, useState } from "react"; +import React, { useCallback, useEffect, useId, useRef, useState } from "react"; interface AdvancedDatePickerProps { value: DateRangePickerValue; @@ -13,6 +13,7 @@ interface AdvancedDatePickerProps { className?: string; showTimeRange?: boolean; align?: "left" | "right"; + compact?: boolean; } export interface RelativeTimeOption { @@ -84,7 +85,9 @@ const AdvancedDatePicker: React.FC = ({ className, showTimeRange = true, align = "right", + compact = false, }) => { + const inputId = useId(); const [isOpen, setIsOpen] = useState(false); const [tempValue, setTempValue] = useState(value); const [selectedOption, setSelectedOption] = useState(null); @@ -279,21 +282,28 @@ const AdvancedDatePicker: React.FC = ({ }; return ( -
    +
    {label &&

    {label}

    } -
    +
    {/* Main input display */}
    {children}
    - - ), - th: ({ children }) => ( - {children} - ), - td: ({ children }) => {children}, - }} - > - {content} - -); - -const UsageAIChatPanel: React.FC = ({ open, onClose, accessToken }) => { - const [messages, setMessages] = useState([]); - const [inputText, setInputText] = useState(""); - const [isLoading, setIsLoading] = useState(false); - const [selectedModel, setSelectedModel] = useState(undefined); - const [availableModels, setAvailableModels] = useState([]); - const [isLoadingModels, setIsLoadingModels] = useState(false); - const [streamingContent, setStreamingContent] = useState(""); - const [statusMessage, setStatusMessage] = useState(null); - const [activeToolCalls, setActiveToolCalls] = useState([]); - const messagesEndRef = useRef(null); - const abortControllerRef = useRef(null); - - useEffect(() => { - if (open && availableModels.length === 0) { - loadModels(); - } - }, [open]); - - useEffect(() => { - if (typeof messagesEndRef.current?.scrollIntoView === "function") { - messagesEndRef.current.scrollIntoView({ behavior: "smooth" }); - } - }, [messages, streamingContent, activeToolCalls, statusMessage]); - - const loadModels = async () => { - if (!accessToken) return; - setIsLoadingModels(true); - try { - const fetchedModels = await modelHubCall(accessToken); - if (fetchedModels?.data?.length > 0) { - const models = fetchedModels.data.map((item: any) => item.model_group as string).sort(); - setAvailableModels(models); - } - } catch (error) { - console.error("Failed to load models:", error); - } finally { - setIsLoadingModels(false); - } - }; - - const handleSend = async () => { - if (!accessToken || !inputText.trim() || isLoading) return; - - const userMessage: ChatMessage = { role: "user", content: inputText.trim() }; - const updatedMessages = [...messages, userMessage]; - setMessages(updatedMessages); - setInputText(""); - setIsLoading(true); - setStreamingContent(""); - setStatusMessage(null); - setActiveToolCalls([]); - - const abortController = new AbortController(); - abortControllerRef.current = abortController; - - let accumulated = ""; - const toolCalls: ToolCallStep[] = []; - - try { - await usageAiChatStream( - accessToken, - updatedMessages.slice(-20).map((m) => ({ role: m.role, content: m.content })), - selectedModel || "", - (content: string) => { - setStatusMessage(null); - accumulated += content; - setStreamingContent(accumulated); - }, - () => { - setStatusMessage(null); - setActiveToolCalls([]); - setMessages((prev) => [ - ...prev, - { role: "assistant", content: accumulated, toolCalls: toolCalls.length > 0 ? [...toolCalls] : undefined }, - ]); - setStreamingContent(""); - }, - (errorMsg: string) => { - setStatusMessage(null); - setActiveToolCalls([]); - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]); - setStreamingContent(""); - }, - (status: string) => { - setStatusMessage(status); - }, - (event: UsageAiToolCallEvent) => { - const idx = toolCalls.findIndex((tc) => tc.tool_name === event.tool_name); - if (idx >= 0) { - toolCalls[idx] = { ...event }; - } else { - toolCalls.push({ ...event }); - } - setActiveToolCalls([...toolCalls]); - }, - abortController.signal, - ); - } catch (error: any) { - if (error?.name === "AbortError" || abortController.signal.aborted) { - return; - } - const errorMsg = error?.message || "Failed to get response. Please try again."; - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]); - setStreamingContent(""); - } finally { - setIsLoading(false); - abortControllerRef.current = null; - } - }; - - const handleKeyDown = (e: React.KeyboardEvent) => { - if (e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - handleSend(); - } - }; - - const handleClose = () => { - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - } - onClose(); - }; - - const handleClear = () => { - setMessages([]); - setStreamingContent(""); - setActiveToolCalls([]); - setStatusMessage(null); - }; - - return ( -
    - {/* Header */} -
    -
    -

    -

    -

    Ask about your spend, models, keys, and trends

    -
    - -
    - - {/* Model selector */} -
    - setSelectedModel(value ?? undefined)} - > - - - {isLoadingModels ? "Loading models…" : "No models found"} - - {(model: string) => ( - - {model} - - )} - - - -
    - - {/* Chat messages */} -
    - {messages.length === 0 && !streamingContent && !isLoading && ( -
    -
    - )} - - {messages.map((msg, idx) => ( -
    - {msg.role === "user" ? ( -
    -
    - {msg.content} -
    -
    - ) : ( -
    - {/* Tool calls for this message */} - {msg.toolCalls && msg.toolCalls.length > 0 && ( -
    - {msg.toolCalls.map((tc, tcIdx) => ( - - ))} -
    - )} - {/* Response */} -
    - -
    -
    - )} -
    - ))} - - {/* Active tool calls (in-progress) */} - {isLoading && activeToolCalls.length > 0 && ( -
    - {activeToolCalls.map((tc, idx) => ( - - ))} -
    - )} - - {/* Status / spinner */} - {isLoading && !streamingContent && ( -
    - - {statusMessage || "Thinking..."} -
    - )} - - {/* Streaming response */} - {streamingContent && ( -
    - -
    - )} - -
    -
    - - {/* Input area */} -
    -
    -