mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge e8c727a110 into e16d79d635
This commit is contained in:
commit
952fa47742
5 changed files with 313 additions and 13 deletions
|
|
@ -46,21 +46,23 @@ class MCPCostCalculator:
|
|||
)
|
||||
or {}
|
||||
)
|
||||
mcp_server_cost_info: Final[MCPServerCostInfo] = (
|
||||
mcp_tool_call_metadata.get("mcp_server_cost_info") or MCPServerCostInfo()
|
||||
return MCPCostCalculator.configured_cost_per_query(
|
||||
mcp_server_cost_info=mcp_tool_call_metadata.get("mcp_server_cost_info"),
|
||||
tool_name=mcp_tool_call_metadata.get("name", ""),
|
||||
)
|
||||
#########################################################
|
||||
# User defined cost per query
|
||||
#########################################################
|
||||
default_cost_per_query: Final = mcp_server_cost_info.get("default_cost_per_query", None)
|
||||
tool_name_to_cost_per_query: Final[dict] = mcp_server_cost_info.get("tool_name_to_cost_per_query", {}) or {}
|
||||
tool_name: Final = mcp_tool_call_metadata.get("name", "")
|
||||
|
||||
#########################################################
|
||||
# 1. If tool_name is in tool_name_to_cost_per_query, use the cost per query
|
||||
# 2. If tool_name is not in tool_name_to_cost_per_query, use the default cost per query
|
||||
# 3. Default to 0.0 if no cost per query is found
|
||||
#########################################################
|
||||
@staticmethod
|
||||
def configured_cost_per_query(mcp_server_cost_info: MCPServerCostInfo | None, tool_name: str) -> float:
|
||||
"""
|
||||
The operator-configured price of one call to ``tool_name``.
|
||||
|
||||
1. If tool_name is in tool_name_to_cost_per_query, use the cost per query
|
||||
2. If tool_name is not in tool_name_to_cost_per_query, use the default cost per query
|
||||
3. Default to 0.0 if no cost per query is found
|
||||
"""
|
||||
cost_info: Final[MCPServerCostInfo] = mcp_server_cost_info or MCPServerCostInfo()
|
||||
default_cost_per_query: Final = cost_info.get("default_cost_per_query", None)
|
||||
tool_name_to_cost_per_query: Final[dict] = cost_info.get("tool_name_to_cost_per_query", {}) or {}
|
||||
cost_per_query: float = 0.0
|
||||
if tool_name in tool_name_to_cost_per_query:
|
||||
cost_per_query = tool_name_to_cost_per_query[tool_name]
|
||||
|
|
|
|||
|
|
@ -322,6 +322,8 @@ async def reserve_budget_for_request(
|
|||
"callback_bound": False,
|
||||
"input_cost": min(float(input_cost or 0.0), reservation_cost),
|
||||
"input_tokens": max(input_token_counts.values(), default=None),
|
||||
"fail_closed_budget_enforcement": fail_closed_budget_enforcement,
|
||||
"counter_limits": _counter_limits(counters=counters, entries=applied_entries),
|
||||
}
|
||||
_start_reservation_lease_renewal(
|
||||
budget_reservation=budget_reservation,
|
||||
|
|
@ -330,6 +332,100 @@ async def reserve_budget_for_request(
|
|||
return budget_reservation
|
||||
|
||||
|
||||
def _counter_limits(
|
||||
counters: Sequence[_BudgetCounter], entries: Sequence[Mapping[str, object]]
|
||||
) -> list[dict[str, float | str | None]]:
|
||||
reserved: Final = {entry.get("counter_key") for entry in entries}
|
||||
return [
|
||||
{
|
||||
"counter_key": counter.counter_key,
|
||||
"max_budget": counter.max_budget,
|
||||
"fallback_spend": counter.fallback_spend,
|
||||
"entity_type": counter.entity_type,
|
||||
"entity_id": counter.entity_id,
|
||||
"source_cache_key": counter.source_cache_key,
|
||||
"spend_log_entity_id": counter.spend_log_entity_id,
|
||||
"window_duration": counter.window_duration,
|
||||
"window_start": counter.window_start.isoformat() if counter.window_start is not None else None,
|
||||
}
|
||||
for counter in counters
|
||||
if counter.counter_key in reserved
|
||||
]
|
||||
|
||||
|
||||
def _counter_from_limit(limit: object) -> _BudgetCounter | None:
|
||||
if not isinstance(limit, Mapping):
|
||||
return None
|
||||
counter_key: Final = limit.get("counter_key")
|
||||
max_budget: Final = _to_float(limit.get("max_budget"))
|
||||
if not isinstance(counter_key, str) or max_budget is None:
|
||||
return None
|
||||
return _BudgetCounter(
|
||||
counter_key=counter_key,
|
||||
max_budget=max_budget,
|
||||
fallback_spend=_to_float(limit.get("fallback_spend")) or 0.0,
|
||||
entity_type=str(limit.get("entity_type") or ""),
|
||||
entity_id=str(limit.get("entity_id") or ""),
|
||||
source_cache_key=_optional_str(limit.get("source_cache_key")),
|
||||
spend_log_entity_id=_optional_str(limit.get("spend_log_entity_id")),
|
||||
window_duration=_optional_str(limit.get("window_duration")),
|
||||
window_start=_coerce_datetime(limit.get("window_start")),
|
||||
)
|
||||
|
||||
|
||||
def _optional_str(value: object) -> str | None:
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
async def reserve_budget_for_tool_call(
|
||||
parent_reservation: Mapping[str, object] | None,
|
||||
tool_cost: float,
|
||||
valid_token: UserAPIKeyAuth | None,
|
||||
) -> dict | None:
|
||||
"""Reserves the price of one tool call the gateway runs for an admitted request, against the counters the
|
||||
request was admitted on and under the same over-budget policy as the request itself."""
|
||||
if not parent_reservation or tool_cost <= 0:
|
||||
return None
|
||||
limits: Final = parent_reservation.get("counter_limits")
|
||||
counters: Final = tuple(
|
||||
counter
|
||||
for counter in map(_counter_from_limit, limits if isinstance(limits, list) else ())
|
||||
if counter is not None
|
||||
)
|
||||
if not counters:
|
||||
return None
|
||||
fail_closed: Final = parent_reservation.get("fail_closed_budget_enforcement") is True
|
||||
applied_entries: Final[list[dict[str, float | str]]] = []
|
||||
reservation_cost = tool_cost
|
||||
try:
|
||||
with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)):
|
||||
reservable: Final = await _initialize_reservation_counters(
|
||||
counters=counters,
|
||||
fail_closed_budget_enforcement=fail_closed,
|
||||
)
|
||||
reservation_cost = await _reserve_reservable_counters(
|
||||
reservable=reservable,
|
||||
valid_token=valid_token,
|
||||
applied_entries=applied_entries,
|
||||
reservation_cost=reservation_cost,
|
||||
fail_closed_budget_enforcement=fail_closed,
|
||||
)
|
||||
except Exception:
|
||||
await _release_applied_entries_best_effort(entries=applied_entries, default_reserved_cost=reservation_cost)
|
||||
raise
|
||||
if not applied_entries:
|
||||
return None
|
||||
return {
|
||||
"reserved_cost": reservation_cost,
|
||||
"entries": applied_entries,
|
||||
"finalized": False,
|
||||
"callback_bound": False,
|
||||
"input_cost": 0.0,
|
||||
"input_tokens": None,
|
||||
"fail_closed_budget_enforcement": fail_closed,
|
||||
}
|
||||
|
||||
|
||||
async def reconcile_budget_reservation(
|
||||
budget_reservation: dict | None,
|
||||
actual_cost: float | None,
|
||||
|
|
|
|||
|
|
@ -736,6 +736,7 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
for tool_call in tool_calls:
|
||||
logging_request_data: dict[str, object] = {}
|
||||
tool_name: str | None = None
|
||||
tool_budget_reservation: dict | None = None
|
||||
try:
|
||||
(
|
||||
tool_name,
|
||||
|
|
@ -820,6 +821,14 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
if user_identifier:
|
||||
logging_request_data["user"] = user_identifier
|
||||
|
||||
# the parent reservation only covers the model call, so each priced tool call reserves its own
|
||||
tool_budget_reservation = await LiteLLM_Proxy_MCP_Handler._reserve_tool_call_budget(
|
||||
user_api_key_auth=typed_user_api_key_auth,
|
||||
logging_metadata=logging_metadata,
|
||||
mcp_server=mcp_server,
|
||||
tool_name=sanitized_tool_name,
|
||||
)
|
||||
|
||||
litellm_logging_obj: LiteLLMLoggingObj | None = None
|
||||
try:
|
||||
litellm_logging_obj, _ = function_setup(
|
||||
|
|
@ -995,11 +1004,45 @@ class LiteLLM_Proxy_MCP_Handler:
|
|||
"name": tool_name,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
await LiteLLM_Proxy_MCP_Handler._release_tool_call_budget(tool_budget_reservation)
|
||||
|
||||
return tool_results
|
||||
|
||||
_execute_tool_calls = execute_tool_calls
|
||||
|
||||
@staticmethod
|
||||
async def _reserve_tool_call_budget(
|
||||
user_api_key_auth: "UserAPIKeyAuth | None",
|
||||
logging_metadata: dict[str, object],
|
||||
mcp_server: MCPServer | None,
|
||||
tool_name: str,
|
||||
) -> dict | None:
|
||||
from litellm.litellm_core_utils.internal_call_metadata import sanitize_user_api_key_auth
|
||||
from litellm.proxy._experimental.mcp_server.cost_calculator import MCPCostCalculator
|
||||
from litellm.proxy.spend_tracking.budget_reservation import reserve_budget_for_tool_call
|
||||
|
||||
logging_metadata.pop("user_api_key_budget_reservation", None)
|
||||
if "user_api_key_auth" in logging_metadata:
|
||||
logging_metadata["user_api_key_auth"] = sanitize_user_api_key_auth(logging_metadata["user_api_key_auth"])
|
||||
mcp_info: Final = (mcp_server.mcp_info if mcp_server is not None else None) or {}
|
||||
tool_reservation: Final = await reserve_budget_for_tool_call(
|
||||
parent_reservation=getattr(user_api_key_auth, "budget_reservation", None),
|
||||
tool_cost=MCPCostCalculator.configured_cost_per_query(
|
||||
mcp_server_cost_info=mcp_info.get("mcp_server_cost_info"), tool_name=tool_name
|
||||
),
|
||||
valid_token=user_api_key_auth,
|
||||
)
|
||||
if tool_reservation is not None:
|
||||
logging_metadata["user_api_key_budget_reservation"] = tool_reservation
|
||||
return tool_reservation
|
||||
|
||||
@staticmethod
|
||||
async def _release_tool_call_budget(tool_reservation: dict | None) -> None:
|
||||
from litellm.proxy.spend_tracking.budget_reservation import release_or_invalidate_budget_reservation
|
||||
|
||||
await release_or_invalidate_budget_reservation(budget_reservation=tool_reservation)
|
||||
|
||||
@staticmethod
|
||||
def create_follow_up_messages_for_chat(
|
||||
original_messages: list[object],
|
||||
|
|
|
|||
|
|
@ -27,8 +27,10 @@ from litellm.proxy.spend_tracking.budget_reservation import (
|
|||
_get_team_member_budget_counter,
|
||||
estimate_request_max_cost,
|
||||
get_budget_window_start,
|
||||
reconcile_budget_reservation,
|
||||
release_unbound_budget_reservation,
|
||||
reserve_budget_for_request,
|
||||
reserve_budget_for_tool_call,
|
||||
)
|
||||
from litellm.proxy.utils import ProxyLogging
|
||||
from litellm.router import Router
|
||||
|
|
@ -426,3 +428,85 @@ async def test_reservation_holds_cost_against_cached_org_and_project_budgets(spe
|
|||
)
|
||||
def test_budget_window_start_reads_json_encoded_windows(window: str, expected: datetime | None) -> None:
|
||||
assert get_budget_window_start(window) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail_closed", [True, False])
|
||||
async def test_auto_executed_tool_calls_are_each_admitted_against_the_request_budget(
|
||||
spend_counter_cache: DualCache, fail_closed: bool
|
||||
):
|
||||
"""One admitted request cannot carry any number of priced tool calls past its budget. Under strict enforcement a
|
||||
call that does not fit is refused, otherwise the budget is exhausted after at most one partially fitting call."""
|
||||
token: Final = UserAPIKeyAuth(token="hashed-fanout-key", max_budget=1.0, spend=0.95)
|
||||
counter_key: Final = "spend:key:hashed-fanout-key"
|
||||
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.95)
|
||||
parent: Final = await reserve_budget_for_request(
|
||||
request_body={"model": "gpt-4o", "max_tokens": 16, "messages": [{"role": "user", "content": "price 40 SKUs"}]},
|
||||
route="/v1/chat/completions",
|
||||
llm_router=None,
|
||||
valid_token=token,
|
||||
team_object=None,
|
||||
user_object=None,
|
||||
prisma_client=None,
|
||||
user_api_key_cache=UserApiKeyCache(),
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=DualCache()),
|
||||
fail_closed_budget_enforcement=fail_closed,
|
||||
)
|
||||
assert parent is not None
|
||||
model_cost: Final = float(parent["reserved_cost"])
|
||||
await reconcile_budget_reservation(budget_reservation=parent, actual_cost=model_cost)
|
||||
|
||||
admitted = 0
|
||||
for _ in range(40):
|
||||
try:
|
||||
tool_reservation = await reserve_budget_for_tool_call(
|
||||
parent_reservation=parent, tool_cost=0.10, valid_token=token
|
||||
)
|
||||
except litellm.BudgetExceededError:
|
||||
continue
|
||||
assert tool_reservation is not None
|
||||
admitted += 1
|
||||
await reconcile_budget_reservation(budget_reservation=tool_reservation, actual_cost=0.10)
|
||||
|
||||
counter: Final = spend_counter_cache.in_memory_cache.get_cache(key=counter_key)
|
||||
if fail_closed:
|
||||
assert admitted == 0
|
||||
assert counter == pytest.approx(0.95 + model_cost)
|
||||
else:
|
||||
assert admitted == 1
|
||||
assert counter == pytest.approx(0.95 + model_cost + 0.10)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_call_reservation_admits_calls_that_fit(spend_counter_cache: DualCache):
|
||||
token: Final = UserAPIKeyAuth(token="hashed-roomy-key", max_budget=1.0, spend=0.5)
|
||||
counter_key: Final = "spend:key:hashed-roomy-key"
|
||||
spend_counter_cache.in_memory_cache.set_cache(key=counter_key, value=0.5)
|
||||
parent: Final = {
|
||||
"reserved_cost": 0.0,
|
||||
"entries": [],
|
||||
"counter_limits": [
|
||||
{
|
||||
"counter_key": counter_key,
|
||||
"max_budget": 1.0,
|
||||
"fallback_spend": 0.5,
|
||||
"entity_type": "Key",
|
||||
"entity_id": "hashed-roomy-key",
|
||||
}
|
||||
],
|
||||
"finalized": True,
|
||||
"fail_closed_budget_enforcement": True,
|
||||
}
|
||||
admitted = 0
|
||||
for _ in range(10):
|
||||
try:
|
||||
tool_reservation = await reserve_budget_for_tool_call(
|
||||
parent_reservation=parent, tool_cost=0.10, valid_token=token
|
||||
)
|
||||
except litellm.BudgetExceededError:
|
||||
break
|
||||
assert tool_reservation is not None
|
||||
await reconcile_budget_reservation(budget_reservation=tool_reservation, actual_cost=0.10)
|
||||
admitted += 1
|
||||
|
||||
assert admitted == 5
|
||||
assert spend_counter_cache.in_memory_cache.get_cache(key=counter_key) == pytest.approx(1.0)
|
||||
|
|
|
|||
|
|
@ -1575,3 +1575,78 @@ async def test_apply_toolset_permissions_pins_the_auth_to_explicit_grants_only(m
|
|||
assert scoped.object_permission is not None
|
||||
assert scoped.object_permission.mcp_servers == ["srv-1"]
|
||||
assert scoped.object_permission.mcp_tool_permissions == {"srv-1": ["add"]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_tool_calls_does_not_run_priced_tool_calls_the_budget_refuses(monkeypatch):
|
||||
"""A priced tool call the budget refuses is never sent to the MCP server. Each executed call carries its own
|
||||
reservation to the cost callback instead of the parent request's, and one nobody settled is released."""
|
||||
import json
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.spend_tracking import budget_reservation as budget_reservation_module
|
||||
|
||||
call_tool_mock = _setup_mcp_call_environment(monkeypatch)
|
||||
_setup_proxy_logging(monkeypatch)
|
||||
sys.modules["litellm.proxy.proxy_server"].proxy_logging_obj.post_mcp_call_hook = AsyncMock(
|
||||
side_effect=lambda response, **_kwargs: response
|
||||
)
|
||||
priced_server = types.SimpleNamespace(
|
||||
alias=None,
|
||||
server_name="price_server",
|
||||
server_id="price-server-id",
|
||||
short_prefix=None,
|
||||
tool_name_to_display_name=None,
|
||||
mcp_info={"mcp_server_cost_info": {"tool_name_to_cost_per_query": {"lookup_price": 0.10}}},
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server import mcp_server_manager as _msm
|
||||
|
||||
_msm.global_mcp_server_manager.get_mcp_server_by_name = MagicMock(return_value=priced_server)
|
||||
|
||||
captured_metadata: list[dict] = []
|
||||
|
||||
def fake_function_setup(*_args, **kwargs):
|
||||
captured_metadata.append(dict(kwargs["metadata"]))
|
||||
raise RuntimeError("no logging object in this test")
|
||||
|
||||
monkeypatch.setattr(mcp_handler_module, "function_setup", fake_function_setup)
|
||||
|
||||
reserved_costs: list[float] = []
|
||||
tool_reservations: list[dict] = []
|
||||
|
||||
async def fake_reserve(parent_reservation, tool_cost, valid_token):
|
||||
assert parent_reservation is parent
|
||||
if len(reserved_costs) == 2:
|
||||
raise litellm.BudgetExceededError(current_cost=1.0, max_budget=1.0, message="Budget has been exceeded!")
|
||||
reserved_costs.append(tool_cost)
|
||||
tool_reservations.append({"reserved_cost": tool_cost, "entries": [], "finalized": False})
|
||||
return tool_reservations[-1]
|
||||
|
||||
monkeypatch.setattr(budget_reservation_module, "reserve_budget_for_tool_call", fake_reserve, raising=False)
|
||||
|
||||
parent: Final = {"reserved_cost": 0.01, "entries": [], "finalized": False}
|
||||
auth: Final = UserAPIKeyAuth(api_key="sk-test", token="hashed", max_budget=1.0, spend=0.75)
|
||||
auth.budget_reservation = parent
|
||||
tool_name = "price_server-lookup_price"
|
||||
tool_calls = [
|
||||
{"id": f"call-{i}", "function": {"name": tool_name, "arguments": json.dumps({"sku": f"SKU-{i}"})}}
|
||||
for i in range(5)
|
||||
]
|
||||
|
||||
results = await LiteLLM_Proxy_MCP_Handler._execute_tool_calls(
|
||||
tool_server_map={tool_name: "price_server"},
|
||||
tool_calls=tool_calls,
|
||||
user_api_key_auth=auth,
|
||||
)
|
||||
|
||||
assert call_tool_mock.await_count == 2
|
||||
assert reserved_costs == [0.10, 0.10]
|
||||
assert [reservation["finalized"] for reservation in tool_reservations] == [True, True]
|
||||
assert len(results) == 5
|
||||
assert [r["result"] for r in results[:2]] == ["Tool executed successfully"] * 2
|
||||
assert all("Budget has been exceeded" in str(r["result"]) for r in results[2:])
|
||||
for metadata in captured_metadata:
|
||||
assert metadata.get("user_api_key_budget_reservation") is not parent
|
||||
nested_auth = metadata.get("user_api_key_auth")
|
||||
assert getattr(nested_auth, "budget_reservation", None) is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue