diff --git a/docs/my-website/docs/proxy/team_budgets.md b/docs/my-website/docs/proxy/team_budgets.md new file mode 100644 index 00000000000..d5675ecc0ee --- /dev/null +++ b/docs/my-website/docs/proxy/team_budgets.md @@ -0,0 +1,128 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# 💰 Setting Team Budgets + +Track spend, set budgets for your Internal Team + +## Setting Monthly Team Budgets + +### 1. Create a team +- Set `max_budget=000000001` ($ value the team is allowed to spend) +- Set `budget_duration="1d"` (How frequently the budget should update) + + + + +Create a new team and set `max_budget` and `budget_duration` +```shell +curl -X POST 'http://0.0.0.0:4000/team/new' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{ + "team_alias": "QA Prod Bot", + "max_budget": 0.000000001, + "budget_duration": "1d" + }' +``` + +Response +```shell +{ + "team_alias": "QA Prod Bot", + "team_id": "de35b29e-6ca8-4f47-b804-2b79d07aa99a", + "max_budget": 0.0001, + "budget_duration": "1d", + "budget_reset_at": "2024-06-14T22:48:36.594000Z" +} +``` + + + + + + +Possible values for `budget_duration` + +| `budget_duration` | When Budget will reset | +| --- | --- | +| `budget_duration="1s"` | every 1 second | +| `budget_duration="1m"` | every 1 min | +| `budget_duration="1h"` | every 1 hour | +| `budget_duration="1d"` | every 1 day | +| `budget_duration="30d"` | every 30 days | + + +### 2. Create a key for the `team` + +Create a key for `team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"` from Step 1 + +💡 **The Budget for Team="QA Prod Bot" budget will apply to this team** + +```shell +curl -X POST 'http://0.0.0.0:4000/key/generate' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{"team_id": "de35b29e-6ca8-4f47-b804-2b79d07aa99a"}' +``` + +Response + +```shell +{"team_id":"de35b29e-6ca8-4f47-b804-2b79d07aa99a", "key":"sk-5qtncoYjzRcxMM4bDRktNQ"} +``` + + +### 3. Test It + +Use the key from step 2 and run this Request twice +```shell +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + -H 'Authorization: Bearer sk-mso-JSykEGri86KyOvgxBw' \ + -H 'Content-Type: application/json' \ + -d ' { + "model": "llama3", + "messages": [ + { + "role": "user", + "content": "hi" + } + ] + }' +``` + +On the 2nd response - expect to see the following exception + +```shell +{ + "error": { + "message": "Budget has been exceeded! Current cost: 3.5e-06, Max budget: 1e-09", + "type": "auth_error", + "param": null, + "code": 400 + } +} +``` + +## Advanced + +### Prometheus metrics for `remaining_budget` + +[More info about Prometheus metrics here](https://docs.litellm.ai/docs/proxy/prometheus) + +You'll need the following in your proxy config.yaml + +```yaml +litellm_settings: + success_callback: ["prometheus"] + failure_callback: ["prometheus"] +``` + +Expect to see this metric on prometheus to track the Remaining Budget for the team + +```shell +litellm_remaining_team_budget_metric{team_alias="QA Prod Bot",team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"} 9.699999999999992e-06 +``` + + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index dc951ea05a7..da9a99a953b 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -44,6 +44,7 @@ const sidebars = { "proxy/self_serve", "proxy/users", "proxy/customers", + "proxy/team_budgets", "proxy/billing", "proxy/user_keys", "proxy/virtual_keys", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 574f7b50ec9..1a09b7a50ea 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6644,7 +6644,7 @@ async def generate_key_fn( # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: - _updated_values = json.dumps(response) + _updated_values = json.dumps(response, default=str) asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( @@ -6749,10 +6749,10 @@ async def update_key_fn( # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: - _updated_values = json.dumps(data_json) + _updated_values = json.dumps(data_json, default=str) _before_value = existing_key_row.json(exclude_none=True) - _before_value = json.dumps(_before_value) + _before_value = json.dumps(_before_value, default=str) asyncio.create_task( create_audit_log_for_update( @@ -6848,7 +6848,7 @@ async def delete_key_fn( ) key_row = key_row.json(exclude_none=True) - _key_row = json.dumps(key_row) + _key_row = json.dumps(key_row, default=str) asyncio.create_task( create_audit_log_for_update( @@ -9964,6 +9964,7 @@ async def new_team( - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget + - budget_duration: Optional[str] - The budget duration for this team - Example "1s", "1d", "1m", "1y" - models: Optional[list] - A list of models associated with the team - all keys for this team_id will have at most, these models. If empty, assumes all models are allowed. - blocked: bool - Flag indicating if the team is blocked or not - will stop all calls from keys with this team_id. @@ -10117,7 +10118,8 @@ async def new_team( # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: _updated_values = complete_team_data.json(exclude_none=True) - _updated_values = json.dumps(_updated_values) + + _updated_values = json.dumps(_updated_values, default=str) asyncio.create_task( create_audit_log_for_update( @@ -10256,8 +10258,8 @@ async def update_team( # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: _before_value = existing_team_row.json(exclude_none=True) - _before_value = json.dumps(_before_value) - _after_value: str = json.dumps(updated_kv) + _before_value = json.dumps(_before_value, default=str) + _after_value: str = json.dumps(updated_kv, default=str) asyncio.create_task( create_audit_log_for_update(