mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-11 22:51:28 +00:00
Merge pull request #4189 from BerriAI/doc_setting_team_budgets
[Doc] Setting Team budgets
This commit is contained in:
Ishaan Jaff
GitHub
commit
a3dd193e19
3 changed files with 138 additions and 7 deletions
128
docs/my-website/docs/proxy/team_budgets.md
Normal file
128
docs/my-website/docs/proxy/team_budgets.md
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
import Image from '@theme/IdealImage';
|
||||
import Tabs from '@theme/Tabs';
|
||||
import TabItem from '@theme/TabItem';
|
||||
|
||||
# 💰 Setting Team Budgets
|
||||
|
||||
Track spend, set budgets for your Internal Team
|
||||
|
||||
## Setting Monthly Team Budgets
|
||||
|
||||
### 1. Create a team
|
||||
- Set `max_budget=000000001` ($ value the team is allowed to spend)
|
||||
- Set `budget_duration="1d"` (How frequently the budget should update)
|
||||
|
||||
<Tabs>
|
||||
<TabItem value="api" label="API">
|
||||
|
||||
Create a new team and set `max_budget` and `budget_duration`
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/team/new' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"team_alias": "QA Prod Bot",
|
||||
"max_budget": 0.000000001,
|
||||
"budget_duration": "1d"
|
||||
}'
|
||||
```
|
||||
|
||||
Response
|
||||
```shell
|
||||
{
|
||||
"team_alias": "QA Prod Bot",
|
||||
"team_id": "de35b29e-6ca8-4f47-b804-2b79d07aa99a",
|
||||
"max_budget": 0.0001,
|
||||
"budget_duration": "1d",
|
||||
"budget_reset_at": "2024-06-14T22:48:36.594000Z"
|
||||
}
|
||||
```
|
||||
</TabItem>
|
||||
<TabItem value="ui" label="UI">
|
||||
</TabItem>
|
||||
</Tabs>
|
||||
|
||||
|
||||
Possible values for `budget_duration`
|
||||
|
||||
| `budget_duration` | When Budget will reset |
|
||||
| --- | --- |
|
||||
| `budget_duration="1s"` | every 1 second |
|
||||
| `budget_duration="1m"` | every 1 min |
|
||||
| `budget_duration="1h"` | every 1 hour |
|
||||
| `budget_duration="1d"` | every 1 day |
|
||||
| `budget_duration="30d"` | every 30 days |
|
||||
|
||||
|
||||
### 2. Create a key for the `team`
|
||||
|
||||
Create a key for `team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"` from Step 1
|
||||
|
||||
💡 **The Budget for Team="QA Prod Bot" budget will apply to this team**
|
||||
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/key/generate' \
|
||||
-H 'Authorization: Bearer sk-1234' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"team_id": "de35b29e-6ca8-4f47-b804-2b79d07aa99a"}'
|
||||
```
|
||||
|
||||
Response
|
||||
|
||||
```shell
|
||||
{"team_id":"de35b29e-6ca8-4f47-b804-2b79d07aa99a", "key":"sk-5qtncoYjzRcxMM4bDRktNQ"}
|
||||
```
|
||||
|
||||
|
||||
### 3. Test It
|
||||
|
||||
Use the key from step 2 and run this Request twice
|
||||
```shell
|
||||
curl -X POST 'http://0.0.0.0:4000/chat/completions' \
|
||||
-H 'Authorization: Bearer sk-mso-JSykEGri86KyOvgxBw' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d ' {
|
||||
"model": "llama3",
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": "hi"
|
||||
}
|
||||
]
|
||||
}'
|
||||
```
|
||||
|
||||
On the 2nd response - expect to see the following exception
|
||||
|
||||
```shell
|
||||
{
|
||||
"error": {
|
||||
"message": "Budget has been exceeded! Current cost: 3.5e-06, Max budget: 1e-09",
|
||||
"type": "auth_error",
|
||||
"param": null,
|
||||
"code": 400
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Advanced
|
||||
|
||||
### Prometheus metrics for `remaining_budget`
|
||||
|
||||
[More info about Prometheus metrics here](https://docs.litellm.ai/docs/proxy/prometheus)
|
||||
|
||||
You'll need the following in your proxy config.yaml
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
success_callback: ["prometheus"]
|
||||
failure_callback: ["prometheus"]
|
||||
```
|
||||
|
||||
Expect to see this metric on prometheus to track the Remaining Budget for the team
|
||||
|
||||
```shell
|
||||
litellm_remaining_team_budget_metric{team_alias="QA Prod Bot",team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"} 9.699999999999992e-06
|
||||
```
|
||||
|
||||
|
||||
|
|
@ -44,6 +44,7 @@ const sidebars = {
|
|||
"proxy/self_serve",
|
||||
"proxy/users",
|
||||
"proxy/customers",
|
||||
"proxy/team_budgets",
|
||||
"proxy/billing",
|
||||
"proxy/user_keys",
|
||||
"proxy/virtual_keys",
|
||||
|
|
|
|||
|
|
@ -6644,7 +6644,7 @@ async def generate_key_fn(
|
|||
|
||||
# Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
|
||||
if litellm.store_audit_logs is True:
|
||||
_updated_values = json.dumps(response)
|
||||
_updated_values = json.dumps(response, default=str)
|
||||
asyncio.create_task(
|
||||
create_audit_log_for_update(
|
||||
request_data=LiteLLM_AuditLogs(
|
||||
|
|
@ -6749,10 +6749,10 @@ async def update_key_fn(
|
|||
|
||||
# Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
|
||||
if litellm.store_audit_logs is True:
|
||||
_updated_values = json.dumps(data_json)
|
||||
_updated_values = json.dumps(data_json, default=str)
|
||||
|
||||
_before_value = existing_key_row.json(exclude_none=True)
|
||||
_before_value = json.dumps(_before_value)
|
||||
_before_value = json.dumps(_before_value, default=str)
|
||||
|
||||
asyncio.create_task(
|
||||
create_audit_log_for_update(
|
||||
|
|
@ -6848,7 +6848,7 @@ async def delete_key_fn(
|
|||
)
|
||||
|
||||
key_row = key_row.json(exclude_none=True)
|
||||
_key_row = json.dumps(key_row)
|
||||
_key_row = json.dumps(key_row, default=str)
|
||||
|
||||
asyncio.create_task(
|
||||
create_audit_log_for_update(
|
||||
|
|
@ -9964,6 +9964,7 @@ async def new_team(
|
|||
- tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit
|
||||
- rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit
|
||||
- max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget
|
||||
- budget_duration: Optional[str] - The budget duration for this team - Example "1s", "1d", "1m", "1y"
|
||||
- models: Optional[list] - A list of models associated with the team - all keys for this team_id will have at most, these models. If empty, assumes all models are allowed.
|
||||
- blocked: bool - Flag indicating if the team is blocked or not - will stop all calls from keys with this team_id.
|
||||
|
||||
|
|
@ -10117,7 +10118,8 @@ async def new_team(
|
|||
# Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
|
||||
if litellm.store_audit_logs is True:
|
||||
_updated_values = complete_team_data.json(exclude_none=True)
|
||||
_updated_values = json.dumps(_updated_values)
|
||||
|
||||
_updated_values = json.dumps(_updated_values, default=str)
|
||||
|
||||
asyncio.create_task(
|
||||
create_audit_log_for_update(
|
||||
|
|
@ -10256,8 +10258,8 @@ async def update_team(
|
|||
# Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True
|
||||
if litellm.store_audit_logs is True:
|
||||
_before_value = existing_team_row.json(exclude_none=True)
|
||||
_before_value = json.dumps(_before_value)
|
||||
_after_value: str = json.dumps(updated_kv)
|
||||
_before_value = json.dumps(_before_value, default=str)
|
||||
_after_value: str = json.dumps(updated_kv, default=str)
|
||||
|
||||
asyncio.create_task(
|
||||
create_audit_log_for_update(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue