diff --git a/litellm/proxy/_experimental/mcp_server/cost_calculator.py b/litellm/proxy/_experimental/mcp_server/cost_calculator.py index e16c9cf62df..b7da38079ed 100644 --- a/litellm/proxy/_experimental/mcp_server/cost_calculator.py +++ b/litellm/proxy/_experimental/mcp_server/cost_calculator.py @@ -46,21 +46,23 @@ class MCPCostCalculator: ) or {} ) - mcp_server_cost_info: Final[MCPServerCostInfo] = ( - mcp_tool_call_metadata.get("mcp_server_cost_info") or MCPServerCostInfo() + return MCPCostCalculator.configured_cost_per_query( + mcp_server_cost_info=mcp_tool_call_metadata.get("mcp_server_cost_info"), + tool_name=mcp_tool_call_metadata.get("name", ""), ) - ######################################################### - # User defined cost per query - ######################################################### - default_cost_per_query: Final = mcp_server_cost_info.get("default_cost_per_query", None) - tool_name_to_cost_per_query: Final[dict] = mcp_server_cost_info.get("tool_name_to_cost_per_query", {}) or {} - tool_name: Final = mcp_tool_call_metadata.get("name", "") - ######################################################### - # 1. If tool_name is in tool_name_to_cost_per_query, use the cost per query - # 2. If tool_name is not in tool_name_to_cost_per_query, use the default cost per query - # 3. Default to 0.0 if no cost per query is found - ######################################################### + @staticmethod + def configured_cost_per_query(mcp_server_cost_info: MCPServerCostInfo | None, tool_name: str) -> float: + """ + The operator-configured price of one call to ``tool_name``. + + 1. If tool_name is in tool_name_to_cost_per_query, use the cost per query + 2. If tool_name is not in tool_name_to_cost_per_query, use the default cost per query + 3. Default to 0.0 if no cost per query is found + """ + cost_info: Final[MCPServerCostInfo] = mcp_server_cost_info or MCPServerCostInfo() + default_cost_per_query: Final = cost_info.get("default_cost_per_query", None) + tool_name_to_cost_per_query: Final[dict] = cost_info.get("tool_name_to_cost_per_query", {}) or {} cost_per_query: float = 0.0 if tool_name in tool_name_to_cost_per_query: cost_per_query = tool_name_to_cost_per_query[tool_name] diff --git a/litellm/proxy/spend_tracking/budget_reservation.py b/litellm/proxy/spend_tracking/budget_reservation.py index 0b91e57f01e..8ab280b3241 100644 --- a/litellm/proxy/spend_tracking/budget_reservation.py +++ b/litellm/proxy/spend_tracking/budget_reservation.py @@ -322,6 +322,8 @@ async def reserve_budget_for_request( "callback_bound": False, "input_cost": min(float(input_cost or 0.0), reservation_cost), "input_tokens": max(input_token_counts.values(), default=None), + "fail_closed_budget_enforcement": fail_closed_budget_enforcement, + "counter_limits": _counter_limits(counters=counters, entries=applied_entries), } _start_reservation_lease_renewal( budget_reservation=budget_reservation, @@ -330,6 +332,100 @@ async def reserve_budget_for_request( return budget_reservation +def _counter_limits( + counters: Sequence[_BudgetCounter], entries: Sequence[Mapping[str, object]] +) -> list[dict[str, float | str | None]]: + reserved: Final = {entry.get("counter_key") for entry in entries} + return [ + { + "counter_key": counter.counter_key, + "max_budget": counter.max_budget, + "fallback_spend": counter.fallback_spend, + "entity_type": counter.entity_type, + "entity_id": counter.entity_id, + "source_cache_key": counter.source_cache_key, + "spend_log_entity_id": counter.spend_log_entity_id, + "window_duration": counter.window_duration, + "window_start": counter.window_start.isoformat() if counter.window_start is not None else None, + } + for counter in counters + if counter.counter_key in reserved + ] + + +def _counter_from_limit(limit: object) -> _BudgetCounter | None: + if not isinstance(limit, Mapping): + return None + counter_key: Final = limit.get("counter_key") + max_budget: Final = _to_float(limit.get("max_budget")) + if not isinstance(counter_key, str) or max_budget is None: + return None + return _BudgetCounter( + counter_key=counter_key, + max_budget=max_budget, + fallback_spend=_to_float(limit.get("fallback_spend")) or 0.0, + entity_type=str(limit.get("entity_type") or ""), + entity_id=str(limit.get("entity_id") or ""), + source_cache_key=_optional_str(limit.get("source_cache_key")), + spend_log_entity_id=_optional_str(limit.get("spend_log_entity_id")), + window_duration=_optional_str(limit.get("window_duration")), + window_start=_coerce_datetime(limit.get("window_start")), + ) + + +def _optional_str(value: object) -> str | None: + return value if isinstance(value, str) else None + + +async def reserve_budget_for_tool_call( + parent_reservation: Mapping[str, object] | None, + tool_cost: float, + valid_token: UserAPIKeyAuth | None, +) -> dict | None: + """Reserves the price of one tool call the gateway runs for an admitted request, against the counters the + request was admitted on and under the same over-budget policy as the request itself.""" + if not parent_reservation or tool_cost <= 0: + return None + limits: Final = parent_reservation.get("counter_limits") + counters: Final = tuple( + counter + for counter in map(_counter_from_limit, limits if isinstance(limits, list) else ()) + if counter is not None + ) + if not counters: + return None + fail_closed: Final = parent_reservation.get("fail_closed_budget_enforcement") is True + applied_entries: Final[list[dict[str, float | str]]] = [] + reservation_cost = tool_cost + try: + with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)): + reservable: Final = await _initialize_reservation_counters( + counters=counters, + fail_closed_budget_enforcement=fail_closed, + ) + reservation_cost = await _reserve_reservable_counters( + reservable=reservable, + valid_token=valid_token, + applied_entries=applied_entries, + reservation_cost=reservation_cost, + fail_closed_budget_enforcement=fail_closed, + ) + except Exception: + await _release_applied_entries_best_effort(entries=applied_entries, default_reserved_cost=reservation_cost) + raise + if not applied_entries: + return None + return { + "reserved_cost": reservation_cost, + "entries": applied_entries, + "finalized": False, + "callback_bound": False, + "input_cost": 0.0, + "input_tokens": None, + "fail_closed_budget_enforcement": fail_closed, + } + + async def reconcile_budget_reservation( budget_reservation: dict | None, actual_cost: float | None, diff --git a/litellm/responses/mcp/litellm_proxy_mcp_handler.py b/litellm/responses/mcp/litellm_proxy_mcp_handler.py index e3c3ab671ea..3db836ccd63 100644 --- a/litellm/responses/mcp/litellm_proxy_mcp_handler.py +++ b/litellm/responses/mcp/litellm_proxy_mcp_handler.py @@ -736,6 +736,7 @@ class LiteLLM_Proxy_MCP_Handler: for tool_call in tool_calls: logging_request_data: dict[str, object] = {} tool_name: str | None = None + tool_budget_reservation: dict | None = None try: ( tool_name, @@ -820,6 +821,14 @@ class LiteLLM_Proxy_MCP_Handler: if user_identifier: logging_request_data["user"] = user_identifier + # the parent reservation only covers the model call, so each priced tool call reserves its own + tool_budget_reservation = await LiteLLM_Proxy_MCP_Handler._reserve_tool_call_budget( + user_api_key_auth=typed_user_api_key_auth, + logging_metadata=logging_metadata, + mcp_server=mcp_server, + tool_name=sanitized_tool_name, + ) + litellm_logging_obj: LiteLLMLoggingObj | None = None try: litellm_logging_obj, _ = function_setup( @@ -995,11 +1004,45 @@ class LiteLLM_Proxy_MCP_Handler: "name": tool_name, } ) + finally: + await LiteLLM_Proxy_MCP_Handler._release_tool_call_budget(tool_budget_reservation) return tool_results _execute_tool_calls = execute_tool_calls + @staticmethod + async def _reserve_tool_call_budget( + user_api_key_auth: "UserAPIKeyAuth | None", + logging_metadata: dict[str, object], + mcp_server: MCPServer | None, + tool_name: str, + ) -> dict | None: + from litellm.litellm_core_utils.internal_call_metadata import sanitize_user_api_key_auth + from litellm.proxy._experimental.mcp_server.cost_calculator import MCPCostCalculator + from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_tool_call + + logging_metadata.pop("user_api_key_budget_reservation", None) + if "user_api_key_auth" in logging_metadata: + logging_metadata["user_api_key_auth"] = sanitize_user_api_key_auth(logging_metadata["user_api_key_auth"]) + mcp_info: Final = (mcp_server.mcp_info if mcp_server is not None else None) or {} + tool_reservation: Final = await reserve_budget_for_tool_call( + parent_reservation=getattr(user_api_key_auth, "budget_reservation", None), + tool_cost=MCPCostCalculator.configured_cost_per_query( + mcp_server_cost_info=mcp_info.get("mcp_server_cost_info"), tool_name=tool_name + ), + valid_token=user_api_key_auth, + ) + if tool_reservation is not None: + logging_metadata["user_api_key_budget_reservation"] = tool_reservation + return tool_reservation + + @staticmethod + async def _release_tool_call_budget(tool_reservation: dict | None) -> None: + from litellm.proxy.spend_tracking.budget_reservation import release_or_invalidate_budget_reservation + + await release_or_invalidate_budget_reservation(budget_reservation=tool_reservation) + @staticmethod def create_follow_up_messages_for_chat( original_messages: list[object], diff --git a/tests/unit/proxy/spend_tracking/test_budget_reservation.py b/tests/unit/proxy/spend_tracking/test_budget_reservation.py index 024fe5229be..1c57c4bace9 100644 --- a/tests/unit/proxy/spend_tracking/test_budget_reservation.py +++ b/tests/unit/proxy/spend_tracking/test_budget_reservation.py @@ -27,8 +27,10 @@ from litellm.proxy.spend_tracking.budget_reservation import ( _get_team_member_budget_counter, estimate_request_max_cost, get_budget_window_start, + reconcile_budget_reservation, release_unbound_budget_reservation, reserve_budget_for_request, + reserve_budget_for_tool_call, ) from litellm.proxy.utils import ProxyLogging from litellm.router import Router @@ -426,3 +428,85 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe ) def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None: assert get_budget_window_start(window) == expected + + +@pytest.mark.parametrize("fail_closed", [True, False]) +async def test_auto_executed_tool_calls_are_each_admitted_against_the_request_budget( + spend_counter_cache: DualCache, fail_closed: bool +): + """One admitted request cannot carry any number of priced tool calls past its budget. Under strict enforcement a + call that does not fit is refused, otherwise the budget is exhausted after at most one partially fitting call.""" + token: Final = UserAPIKeyAuth(token="hashed-fanout-key", max_budget=1.0, spend=0.95) + counter_key: Final = "spend:key:hashed-fanout-key" + spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.95) + parent: Final = await reserve_budget_for_request( + request_body={"model": "gpt-4o", "max_tokens": 16, "messages": [{"role": "user", "content": "price 40 SKUs"}]}, + route="/v1/chat/completions", + llm_router=None, + valid_token=token, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=UserApiKeyCache(), + proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()), + fail_closed_budget_enforcement=fail_closed, + ) + assert parent is not None + model_cost: Final = float(parent["reserved_cost"]) + await reconcile_budget_reservation(budget_reservation=parent, actual_cost=model_cost) + + admitted = 0 + for _ in range(40): + try: + tool_reservation = await reserve_budget_for_tool_call( + parent_reservation=parent, tool_cost=0.10, valid_token=token + ) + except litellm.BudgetExceededError: + continue + assert tool_reservation is not None + admitted += 1 + await reconcile_budget_reservation(budget_reservation=tool_reservation, actual_cost=0.10) + + counter: Final = spend_counter_cache.in_memory_cache.get_cache(key=counter_key) + if fail_closed: + assert admitted == 0 + assert counter == pytest.approx(0.95 + model_cost) + else: + assert admitted == 1 + assert counter == pytest.approx(0.95 + model_cost + 0.10) + + +@pytest.mark.asyncio +async def test_tool_call_reservation_admits_calls_that_fit(spend_counter_cache: DualCache): + token: Final = UserAPIKeyAuth(token="hashed-roomy-key", max_budget=1.0, spend=0.5) + counter_key: Final = "spend:key:hashed-roomy-key" + spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.5) + parent: Final = { + "reserved_cost": 0.0, + "entries": [], + "counter_limits": [ + { + "counter_key": counter_key, + "max_budget": 1.0, + "fallback_spend": 0.5, + "entity_type": "Key", + "entity_id": "hashed-roomy-key", + } + ], + "finalized": True, + "fail_closed_budget_enforcement": True, + } + admitted = 0 + for _ in range(10): + try: + tool_reservation = await reserve_budget_for_tool_call( + parent_reservation=parent, tool_cost=0.10, valid_token=token + ) + except litellm.BudgetExceededError: + break + assert tool_reservation is not None + await reconcile_budget_reservation(budget_reservation=tool_reservation, actual_cost=0.10) + admitted += 1 + + assert admitted == 5 + assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(1.0) diff --git a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py index 26553558298..6bcd30d255d 100644 --- a/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py +++ b/tests/unit/responses/mcp/test_litellm_proxy_mcp_handler.py @@ -1575,3 +1575,78 @@ async def test_apply_toolset_permissions_pins_the_auth_to_explicit_grants_only(m assert scoped.object_permission is not None assert scoped.object_permission.mcp_servers == ["srv-1"] assert scoped.object_permission.mcp_tool_permissions == {"srv-1": ["add"]} + + +@pytest.mark.asyncio +async def test_execute_tool_calls_does_not_run_priced_tool_calls_the_budget_refuses(monkeypatch): + """A priced tool call the budget refuses is never sent to the MCP server. Each executed call carries its own + reservation to the cost callback instead of the parent request's, and one nobody settled is released.""" + import json + + import litellm + from litellm.proxy._types import UserAPIKeyAuth + from litellm.proxy.spend_tracking import budget_reservation as budget_reservation_module + + call_tool_mock = _setup_mcp_call_environment(monkeypatch) + _setup_proxy_logging(monkeypatch) + sys.modules["litellm.proxy.proxy_server"].proxy_logging_obj.post_mcp_call_hook = AsyncMock( + side_effect=lambda response, **_kwargs: response + ) + priced_server = types.SimpleNamespace( + alias=None, + server_name="price_server", + server_id="price-server-id", + short_prefix=None, + tool_name_to_display_name=None, + mcp_info={"mcp_server_cost_info": {"tool_name_to_cost_per_query": {"lookup_price": 0.10}}}, + ) + from litellm.proxy._experimental.mcp_server import mcp_server_manager as _msm + + _msm.global_mcp_server_manager.get_mcp_server_by_name = MagicMock(return_value=priced_server) + + captured_metadata: list[dict] = [] + + def fake_function_setup(*_args, **kwargs): + captured_metadata.append(dict(kwargs["metadata"])) + raise RuntimeError("no logging object in this test") + + monkeypatch.setattr(mcp_handler_module, "function_setup", fake_function_setup) + + reserved_costs: list[float] = [] + tool_reservations: list[dict] = [] + + async def fake_reserve(parent_reservation, tool_cost, valid_token): + assert parent_reservation is parent + if len(reserved_costs) == 2: + raise litellm.BudgetExceededError(current_cost=1.0, max_budget=1.0, message="Budget has been exceeded!") + reserved_costs.append(tool_cost) + tool_reservations.append({"reserved_cost": tool_cost, "entries": [], "finalized": False}) + return tool_reservations[-1] + + monkeypatch.setattr(budget_reservation_module, "reserve_budget_for_tool_call", fake_reserve, raising=False) + + parent: Final = {"reserved_cost": 0.01, "entries": [], "finalized": False} + auth: Final = UserAPIKeyAuth(api_key="sk-test", token="hashed", max_budget=1.0, spend=0.75) + auth.budget_reservation = parent + tool_name = "price_server-lookup_price" + tool_calls = [ + {"id": f"call-{i}", "function": {"name": tool_name, "arguments": json.dumps({"sku": f"SKU-{i}"})}} + for i in range(5) + ] + + results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls( + tool_server_map={tool_name: "price_server"}, + tool_calls=tool_calls, + user_api_key_auth=auth, + ) + + assert call_tool_mock.await_count == 2 + assert reserved_costs == [0.10, 0.10] + assert [reservation["finalized"] for reservation in tool_reservations] == [True, True] + assert len(results) == 5 + assert [r["result"] for r in results[:2]] == ["Tool executed successfully"] * 2 + assert all("Budget has been exceeded" in str(r["result"]) for r in results[2:]) + for metadata in captured_metadata: + assert metadata.get("user_api_key_budget_reservation") is not parent + nested_auth = metadata.get("user_api_key_auth") + assert getattr(nested_auth, "budget_reservation", None) is None