This commit is contained in:
tomatomptt 2026-10-27 02:24:31 +00:00 • committed by GitHub
commit 952fa47742
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 313 additions and 13 deletions

View file

@ -46,21 +46,23 @@ class MCPCostCalculator:
)
or {}
)
mcp_server_cost_info: Final[MCPServerCostInfo] = (
mcp_tool_call_metadata.get("mcp_server_cost_info") or MCPServerCostInfo()
return MCPCostCalculator.configured_cost_per_query(
mcp_server_cost_info=mcp_tool_call_metadata.get("mcp_server_cost_info"),
tool_name=mcp_tool_call_metadata.get("name", ""),
)
#########################################################
# User defined cost per query
#########################################################
default_cost_per_query: Final = mcp_server_cost_info.get("default_cost_per_query", None)
tool_name_to_cost_per_query: Final[dict] = mcp_server_cost_info.get("tool_name_to_cost_per_query", {}) or {}
tool_name: Final = mcp_tool_call_metadata.get("name", "")
#########################################################
# 1. If tool_name is in tool_name_to_cost_per_query, use the cost per query
# 2. If tool_name is not in tool_name_to_cost_per_query, use the default cost per query
# 3. Default to 0.0 if no cost per query is found
#########################################################
@staticmethod
def configured_cost_per_query(mcp_server_cost_info: MCPServerCostInfo | None, tool_name: str) -> float:
"""
The operator-configured price of one call to ``tool_name``.
1. If tool_name is in tool_name_to_cost_per_query, use the cost per query
2. If tool_name is not in tool_name_to_cost_per_query, use the default cost per query
3. Default to 0.0 if no cost per query is found
"""
cost_info: Final[MCPServerCostInfo] = mcp_server_cost_info or MCPServerCostInfo()
default_cost_per_query: Final = cost_info.get("default_cost_per_query", None)
tool_name_to_cost_per_query: Final[dict] = cost_info.get("tool_name_to_cost_per_query", {}) or {}
cost_per_query: float = 0.0
if tool_name in tool_name_to_cost_per_query:
cost_per_query = tool_name_to_cost_per_query[tool_name]

View file

@ -322,6 +322,8 @@ async def reserve_budget_for_request(
"callback_bound": False,
"input_cost": min(float(input_cost or 0.0), reservation_cost),
"input_tokens": max(input_token_counts.values(), default=None),
"fail_closed_budget_enforcement": fail_closed_budget_enforcement,
"counter_limits": _counter_limits(counters=counters, entries=applied_entries),
}
_start_reservation_lease_renewal(
budget_reservation=budget_reservation,
@ -330,6 +332,100 @@ async def reserve_budget_for_request(
return budget_reservation
def _counter_limits(
counters: Sequence[_BudgetCounter], entries: Sequence[Mapping[str, object]]
) -> list[dict[str, float | str | None]]:
reserved: Final = {entry.get("counter_key") for entry in entries}
return [
{
"counter_key": counter.counter_key,
"max_budget": counter.max_budget,
"fallback_spend": counter.fallback_spend,
"entity_type": counter.entity_type,
"entity_id": counter.entity_id,
"source_cache_key": counter.source_cache_key,
"spend_log_entity_id": counter.spend_log_entity_id,
"window_duration": counter.window_duration,
"window_start": counter.window_start.isoformat() if counter.window_start is not None else None,
}
for counter in counters
if counter.counter_key in reserved
]
def _counter_from_limit(limit: object) -> _BudgetCounter | None:
if not isinstance(limit, Mapping):
return None
counter_key: Final = limit.get("counter_key")
max_budget: Final = _to_float(limit.get("max_budget"))
if not isinstance(counter_key, str) or max_budget is None:
return None
return _BudgetCounter(
counter_key=counter_key,
max_budget=max_budget,
fallback_spend=_to_float(limit.get("fallback_spend")) or 0.0,
entity_type=str(limit.get("entity_type") or ""),
entity_id=str(limit.get("entity_id") or ""),
source_cache_key=_optional_str(limit.get("source_cache_key")),
spend_log_entity_id=_optional_str(limit.get("spend_log_entity_id")),
window_duration=_optional_str(limit.get("window_duration")),
window_start=_coerce_datetime(limit.get("window_start")),
)
def _optional_str(value: object) -> str | None:
return value if isinstance(value, str) else None
async def reserve_budget_for_tool_call(
parent_reservation: Mapping[str, object] | None,
tool_cost: float,
valid_token: UserAPIKeyAuth | None,
) -> dict | None:
"""Reserves the price of one tool call the gateway runs for an admitted request, against the counters the
request was admitted on and under the same over-budget policy as the request itself."""
if not parent_reservation or tool_cost <= 0:
return None
limits: Final = parent_reservation.get("counter_limits")
counters: Final = tuple(
counter
for counter in map(_counter_from_limit, limits if isinstance(limits, list) else ())
if counter is not None
)
if not counters:
return None
fail_closed: Final = parent_reservation.get("fail_closed_budget_enforcement") is True
applied_entries: Final[list[dict[str, float | str]]] = []
reservation_cost = tool_cost
try:
with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)):
reservable: Final = await _initialize_reservation_counters(
counters=counters,
fail_closed_budget_enforcement=fail_closed,
)
reservation_cost = await _reserve_reservable_counters(
reservable=reservable,
valid_token=valid_token,
applied_entries=applied_entries,
reservation_cost=reservation_cost,
fail_closed_budget_enforcement=fail_closed,
)
except Exception:
await _release_applied_entries_best_effort(entries=applied_entries, default_reserved_cost=reservation_cost)
raise
if not applied_entries:
return None
return {
"reserved_cost": reservation_cost,
"entries": applied_entries,
"finalized": False,
"callback_bound": False,
"input_cost": 0.0,
"input_tokens": None,
"fail_closed_budget_enforcement": fail_closed,
}
async def reconcile_budget_reservation(
budget_reservation: dict | None,
actual_cost: float | None,

View file

@ -736,6 +736,7 @@ class LiteLLM_Proxy_MCP_Handler:
for tool_call in tool_calls:
logging_request_data: dict[str, object] = {}
tool_name: str | None = None
tool_budget_reservation: dict | None = None
try:
(
tool_name,
@ -820,6 +821,14 @@ class LiteLLM_Proxy_MCP_Handler:
if user_identifier:
logging_request_data["user"] = user_identifier
# the parent reservation only covers the model call, so each priced tool call reserves its own
tool_budget_reservation = await LiteLLM_Proxy_MCP_Handler._reserve_tool_call_budget(
user_api_key_auth=typed_user_api_key_auth,
logging_metadata=logging_metadata,
mcp_server=mcp_server,
tool_name=sanitized_tool_name,
)
litellm_logging_obj: LiteLLMLoggingObj | None = None
try:
litellm_logging_obj, _ = function_setup(
@ -995,11 +1004,45 @@ class LiteLLM_Proxy_MCP_Handler:
"name": tool_name,
}
)
finally:
await LiteLLM_Proxy_MCP_Handler._release_tool_call_budget(tool_budget_reservation)
return tool_results
_execute_tool_calls = execute_tool_calls
@staticmethod
async def _reserve_tool_call_budget(
user_api_key_auth: "UserAPIKeyAuth | None",
logging_metadata: dict[str, object],
mcp_server: MCPServer | None,
tool_name: str,
) -> dict | None:
from litellm.litellm_core_utils.internal_call_metadata import sanitize_user_api_key_auth
from litellm.proxy._experimental.mcp_server.cost_calculator import MCPCostCalculator
from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_tool_call
logging_metadata.pop("user_api_key_budget_reservation", None)
if "user_api_key_auth" in logging_metadata:
logging_metadata["user_api_key_auth"] = sanitize_user_api_key_auth(logging_metadata["user_api_key_auth"])
mcp_info: Final = (mcp_server.mcp_info if mcp_server is not None else None) or {}
tool_reservation: Final = await reserve_budget_for_tool_call(
parent_reservation=getattr(user_api_key_auth, "budget_reservation", None),
tool_cost=MCPCostCalculator.configured_cost_per_query(
mcp_server_cost_info=mcp_info.get("mcp_server_cost_info"), tool_name=tool_name
),
valid_token=user_api_key_auth,
)
if tool_reservation is not None:
logging_metadata["user_api_key_budget_reservation"] = tool_reservation
return tool_reservation
@staticmethod
async def _release_tool_call_budget(tool_reservation: dict | None) -> None:
from litellm.proxy.spend_tracking.budget_reservation import release_or_invalidate_budget_reservation
await release_or_invalidate_budget_reservation(budget_reservation=tool_reservation)
@staticmethod
def create_follow_up_messages_for_chat(
original_messages: list[object],

View file

@ -27,8 +27,10 @@ from litellm.proxy.spend_tracking.budget_reservation import (
_get_team_member_budget_counter,
estimate_request_max_cost,
get_budget_window_start,
reconcile_budget_reservation,
release_unbound_budget_reservation,
reserve_budget_for_request,
reserve_budget_for_tool_call,
)
from litellm.proxy.utils import ProxyLogging
from litellm.router import Router
@ -426,3 +428,85 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe
)
def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None:
assert get_budget_window_start(window) == expected
@pytest.mark.parametrize("fail_closed", [True, False])
async def test_auto_executed_tool_calls_are_each_admitted_against_the_request_budget(
spend_counter_cache: DualCache, fail_closed: bool
):
"""One admitted request cannot carry any number of priced tool calls past its budget. Under strict enforcement a
call that does not fit is refused, otherwise the budget is exhausted after at most one partially fitting call."""
token: Final = UserAPIKeyAuth(token="hashed-fanout-key", max_budget=1.0, spend=0.95)
counter_key: Final = "spend:key:hashed-fanout-key"
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.95)
parent: Final = await reserve_budget_for_request(
request_body={"model": "gpt-4o", "max_tokens": 16, "messages": [{"role": "user", "content": "price 40 SKUs"}]},
route="/v1/chat/completions",
llm_router=None,
valid_token=token,
team_object=None,
user_object=None,
prisma_client=None,
user_api_key_cache=UserApiKeyCache(),
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
fail_closed_budget_enforcement=fail_closed,
)
assert parent is not None
model_cost: Final = float(parent["reserved_cost"])
await reconcile_budget_reservation(budget_reservation=parent, actual_cost=model_cost)
admitted = 0
for _ in range(40):
try:
tool_reservation = await reserve_budget_for_tool_call(
parent_reservation=parent, tool_cost=0.10, valid_token=token
)
except litellm.BudgetExceededError:
continue
assert tool_reservation is not None
admitted += 1
await reconcile_budget_reservation(budget_reservation=tool_reservation, actual_cost=0.10)
counter: Final = spend_counter_cache.in_memory_cache.get_cache(key=counter_key)
if fail_closed:
assert admitted == 0
assert counter == pytest.approx(0.95 + model_cost)
else:
assert admitted == 1
assert counter == pytest.approx(0.95 + model_cost + 0.10)
@pytest.mark.asyncio
async def test_tool_call_reservation_admits_calls_that_fit(spend_counter_cache: DualCache):
token: Final = UserAPIKeyAuth(token="hashed-roomy-key", max_budget=1.0, spend=0.5)
counter_key: Final = "spend:key:hashed-roomy-key"
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.5)
parent: Final = {
"reserved_cost": 0.0,
"entries": [],
"counter_limits": [
{
"counter_key": counter_key,
"max_budget": 1.0,
"fallback_spend": 0.5,
"entity_type": "Key",
"entity_id": "hashed-roomy-key",
}
],
"finalized": True,
"fail_closed_budget_enforcement": True,
}
admitted = 0
for _ in range(10):
try:
tool_reservation = await reserve_budget_for_tool_call(
parent_reservation=parent, tool_cost=0.10, valid_token=token
)
except litellm.BudgetExceededError:
break
assert tool_reservation is not None
await reconcile_budget_reservation(budget_reservation=tool_reservation, actual_cost=0.10)
admitted += 1
assert admitted == 5
assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(1.0)

View file

@ -1575,3 +1575,78 @@ async def test_apply_toolset_permissions_pins_the_auth_to_explicit_grants_only(m
assert scoped.object_permission is not None
assert scoped.object_permission.mcp_servers == ["srv-1"]
assert scoped.object_permission.mcp_tool_permissions == {"srv-1": ["add"]}
@pytest.mark.asyncio
async def test_execute_tool_calls_does_not_run_priced_tool_calls_the_budget_refuses(monkeypatch):
"""A priced tool call the budget refuses is never sent to the MCP server. Each executed call carries its own
reservation to the cost callback instead of the parent request's, and one nobody settled is released."""
import json
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.spend_tracking import budget_reservation as budget_reservation_module
call_tool_mock = _setup_mcp_call_environment(monkeypatch)
_setup_proxy_logging(monkeypatch)
sys.modules["litellm.proxy.proxy_server"].proxy_logging_obj.post_mcp_call_hook = AsyncMock(
side_effect=lambda response, **_kwargs: response
)
priced_server = types.SimpleNamespace(
alias=None,
server_name="price_server",
server_id="price-server-id",
short_prefix=None,
tool_name_to_display_name=None,
mcp_info={"mcp_server_cost_info": {"tool_name_to_cost_per_query": {"lookup_price": 0.10}}},
)
from litellm.proxy._experimental.mcp_server import mcp_server_manager as _msm
_msm.global_mcp_server_manager.get_mcp_server_by_name = MagicMock(return_value=priced_server)
captured_metadata: list[dict] = []
def fake_function_setup(*_args, **kwargs):
captured_metadata.append(dict(kwargs["metadata"]))
raise RuntimeError("no logging object in this test")
monkeypatch.setattr(mcp_handler_module, "function_setup", fake_function_setup)
reserved_costs: list[float] = []
tool_reservations: list[dict] = []
async def fake_reserve(parent_reservation, tool_cost, valid_token):
assert parent_reservation is parent
if len(reserved_costs) == 2:
raise litellm.BudgetExceededError(current_cost=1.0, max_budget=1.0, message="Budget has been exceeded!")
reserved_costs.append(tool_cost)
tool_reservations.append({"reserved_cost": tool_cost, "entries": [], "finalized": False})
return tool_reservations[-1]
monkeypatch.setattr(budget_reservation_module, "reserve_budget_for_tool_call", fake_reserve, raising=False)
parent: Final = {"reserved_cost": 0.01, "entries": [], "finalized": False}
auth: Final = UserAPIKeyAuth(api_key="sk-test", token="hashed", max_budget=1.0, spend=0.75)
auth.budget_reservation = parent
tool_name = "price_server-lookup_price"
tool_calls = [
{"id": f"call-{i}", "function": {"name": tool_name, "arguments": json.dumps({"sku": f"SKU-{i}"})}}
for i in range(5)
]
results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
tool_server_map={tool_name: "price_server"},
tool_calls=tool_calls,
user_api_key_auth=auth,
)
assert call_tool_mock.await_count == 2
assert reserved_costs == [0.10, 0.10]
assert [reservation["finalized"] for reservation in tool_reservations] == [True, True]
assert len(results) == 5
assert [r["result"] for r in results[:2]] == ["Tool executed successfully"] * 2
assert all("Budget has been exceeded" in str(r["result"]) for r in results[2:])
for metadata in captured_metadata:
assert metadata.get("user_api_key_budget_reservation") is not parent
nested_auth = metadata.get("user_api_key_auth")
assert getattr(nested_auth, "budget_reservation", None) is None