From ecb859b62ede237760675e554f24928274e08d29 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Jun 2024 15:56:43 -0700 Subject: [PATCH 1/4] doc - setting team budgets --- docs/my-website/docs/proxy/team_budgets.md | 98 ++++++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 99 insertions(+) create mode 100644 docs/my-website/docs/proxy/team_budgets.md diff --git a/docs/my-website/docs/proxy/team_budgets.md b/docs/my-website/docs/proxy/team_budgets.md new file mode 100644 index 00000000000..327bae34d0c --- /dev/null +++ b/docs/my-website/docs/proxy/team_budgets.md @@ -0,0 +1,98 @@ +import Image from '@theme/IdealImage'; +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Managing Team Budgets + +Track spend, set budgets for your Internal Team + +## Setting Monthly Team Budgets + +### 1. Create a team +with `max_budget` and `budget_duration` + + + + +Set `max_budget` and `budget_duration` +```shell +curl --location 'http://0.0.0.0:4000/team/new' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "team_alias": "QA Prod Bot", + "max_budget": 0.000000001, + "budget_duration": "1d" + }' +``` + +Response +```shell +{ + "team_alias": "QA Prod Bot", + "team_id": "de35b29e-6ca8-4f47-b804-2b79d07aa99a", + "max_budget": 0.0001, + "budget_duration": "1d", + "budget_reset_at": "2024-06-14T22:48:36.594000Z" +} +``` + + + + + +### 2. Create a key for the `team` + +```shell +curl --location 'http://0.0.0.0:4000/key/generate' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "team_id": "de35b29e-6ca8-4f47-b804-2b79d07aa99a" + }' +``` + +Response + +```shell +{"team_id":"de35b29e-6ca8-4f47-b804-2b79d07aa99a", "key":"sk-5qtncoYjzRcxMM4bDRktNQ"} +``` + + +### 3. Test It + +Run this Request twice +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Authorization: Bearer sk-mso-JSykEGri86KyOvgxBw' \ +--header 'Content-Type: application/json' \ +--data ' { + "model": "llama3", + "messages": [ + { + "role": "user", + "content": "hi" + } + ], + } +' +``` + +On the 2nd response - expect to see the following exception + +```shell +{ + "error": { + "message": "Budget has been exceeded! Current cost: 3.5e-06, Max budget: 1e-09", + "type": "auth_error", + "param": null, + "code": 400 + } +} +``` + + +### 4. Prometheus metrics for `remaining_budget` + + +## Updating Team Budgets \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index dc951ea05a7..da9a99a953b 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -44,6 +44,7 @@ const sidebars = { "proxy/self_serve", "proxy/users", "proxy/customers", + "proxy/team_budgets", "proxy/billing", "proxy/user_keys", "proxy/virtual_keys", From e0985af6cd7f9aa06834f017df460c95246181dd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Jun 2024 16:24:45 -0700 Subject: [PATCH 2/4] fix /team/update --- litellm/proxy/proxy_server.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ff1036d5214..cca737eed00 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6644,7 +6644,7 @@ async def generate_key_fn( # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: - _updated_values = json.dumps(response) + _updated_values = json.dumps(response, default=str) asyncio.create_task( create_audit_log_for_update( request_data=LiteLLM_AuditLogs( @@ -6749,10 +6749,10 @@ async def update_key_fn( # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: - _updated_values = json.dumps(data_json) + _updated_values = json.dumps(data_json, default=str) _before_value = existing_key_row.json(exclude_none=True) - _before_value = json.dumps(_before_value) + _before_value = json.dumps(_before_value, default=str) asyncio.create_task( create_audit_log_for_update( @@ -6848,7 +6848,7 @@ async def delete_key_fn( ) key_row = key_row.json(exclude_none=True) - _key_row = json.dumps(key_row) + _key_row = json.dumps(key_row, default=str) asyncio.create_task( create_audit_log_for_update( @@ -9964,6 +9964,7 @@ async def new_team( - tpm_limit: Optional[int] - The TPM (Tokens Per Minute) limit for this team - all keys with this team_id will have at max this TPM limit - rpm_limit: Optional[int] - The RPM (Requests Per Minute) limit for this team - all keys associated with this team_id will have at max this RPM limit - max_budget: Optional[float] - The maximum budget allocated to the team - all keys for this team_id will have at max this max_budget + - budget_duration: Optional[str] - The budget duration for this team - Example "1s", "1d", "1m", "1y" - models: Optional[list] - A list of models associated with the team - all keys for this team_id will have at most, these models. If empty, assumes all models are allowed. - blocked: bool - Flag indicating if the team is blocked or not - will stop all calls from keys with this team_id. @@ -10117,7 +10118,8 @@ async def new_team( # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: _updated_values = complete_team_data.json(exclude_none=True) - _updated_values = json.dumps(_updated_values) + + _updated_values = json.dumps(_updated_values, default=str) asyncio.create_task( create_audit_log_for_update( @@ -10255,8 +10257,8 @@ async def update_team( # Enterprise Feature - Audit Logging. Enable with litellm.store_audit_logs = True if litellm.store_audit_logs is True: _before_value = existing_team_row.json(exclude_none=True) - _before_value = json.dumps(_before_value) - _after_value: str = json.dumps(updated_kv) + _before_value = json.dumps(_before_value, default=str) + _after_value: str = json.dumps(updated_kv, default=str) asyncio.create_task( create_audit_log_for_update( From 879797c6ba6fe0e0a7ed6abe65f7e172666a0416 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Jun 2024 16:59:01 -0700 Subject: [PATCH 3/4] doc - team based budgets --- docs/my-website/docs/proxy/team_budgets.md | 94 ++++++++++++++-------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/docs/my-website/docs/proxy/team_budgets.md b/docs/my-website/docs/proxy/team_budgets.md index 327bae34d0c..ff15eeacf34 100644 --- a/docs/my-website/docs/proxy/team_budgets.md +++ b/docs/my-website/docs/proxy/team_budgets.md @@ -2,28 +2,29 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Managing Team Budgets +# Setting Team Budgets Track spend, set budgets for your Internal Team ## Setting Monthly Team Budgets ### 1. Create a team -with `max_budget` and `budget_duration` +- Set `max_budget=000000001` ($ value the team is allowed to spend) +- Set `budget_duration="1d"` (How frequently the budget should update) -Set `max_budget` and `budget_duration` +Create and new team and set `max_budget` and `budget_duration` ```shell -curl --location 'http://0.0.0.0:4000/team/new' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "team_alias": "QA Prod Bot", - "max_budget": 0.000000001, - "budget_duration": "1d" - }' +curl -X POST 'http://0.0.0.0:4000/team/new' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{ + "team_alias": "QA Prod Bot", + "max_budget": 0.000000001, + "budget_duration": "1d" + }' ``` Response @@ -41,15 +42,29 @@ Response + +Possible values for `budget_duration` + +| `budget_duration` | When Budget will reset | +| --- | --- | +| `budget_duration="1s"` | every 1 second | +| `budget_duration="1m"` | every 1 min | +| `budget_duration="1h"` | every 1 hour | +| `budget_duration="1d"` | every 1 day | +| `budget_duration="30d"` | every 30 days | + + ### 2. Create a key for the `team` +Create a key for `team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"` from Step 1 + +💡 **The Budget for Team="QA Prod Bot" budget will apply to this team** + ```shell -curl --location 'http://0.0.0.0:4000/key/generate' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "team_id": "de35b29e-6ca8-4f47-b804-2b79d07aa99a" - }' +curl -X POST 'http://0.0.0.0:4000/key/generate' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{"team_id": "de35b29e-6ca8-4f47-b804-2b79d07aa99a"}' ``` Response @@ -61,21 +76,20 @@ Response ### 3. Test It -Run this Request twice +Use the key from step 2 and run this Request twice ```shell -curl --location 'http://0.0.0.0:4000/chat/completions' \ ---header 'Authorization: Bearer sk-mso-JSykEGri86KyOvgxBw' \ ---header 'Content-Type: application/json' \ ---data ' { - "model": "llama3", - "messages": [ - { - "role": "user", - "content": "hi" - } - ], - } -' +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + -H 'Authorization: Bearer sk-mso-JSykEGri86KyOvgxBw' \ + -H 'Content-Type: application/json' \ + -d ' { + "model": "llama3", + "messages": [ + { + "role": "user", + "content": "hi" + } + ] + }' ``` On the 2nd response - expect to see the following exception @@ -91,8 +105,22 @@ On the 2nd response - expect to see the following exception } ``` +## Advanced -### 4. Prometheus metrics for `remaining_budget` +### Prometheus metrics for `remaining_budget` + +You'll need the following in your proxy config.yaml + +```yaml +litellm_settings: + success_callback: ["prometheus"] + failure_callback: ["prometheus"] +``` + +Expect to see this metric on prometheus to track the Remaining Budget for the team + +```shell +litellm_remaining_team_budget_metric{team_alias="QA Prod Bot",team_id="de35b29e-6ca8-4f47-b804-2b79d07aa99a"} 9.699999999999992e-06 +``` -## Updating Team Budgets \ No newline at end of file From afe2ea341c0ca37b1f1a9d683fa4fb5bdf944283 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 13 Jun 2024 17:01:27 -0700 Subject: [PATCH 4/4] doc - setting team budgets --- docs/my-website/docs/proxy/team_budgets.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/team_budgets.md b/docs/my-website/docs/proxy/team_budgets.md index ff15eeacf34..d5675ecc0ee 100644 --- a/docs/my-website/docs/proxy/team_budgets.md +++ b/docs/my-website/docs/proxy/team_budgets.md @@ -2,7 +2,7 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Setting Team Budgets +# 💰 Setting Team Budgets Track spend, set budgets for your Internal Team @@ -15,7 +15,7 @@ Track spend, set budgets for your Internal Team -Create and new team and set `max_budget` and `budget_duration` +Create a new team and set `max_budget` and `budget_duration` ```shell curl -X POST 'http://0.0.0.0:4000/team/new' \ -H 'Authorization: Bearer sk-1234' \ @@ -109,6 +109,8 @@ On the 2nd response - expect to see the following exception ### Prometheus metrics for `remaining_budget` +[More info about Prometheus metrics here](https://docs.litellm.ai/docs/proxy/prometheus) + You'll need the following in your proxy config.yaml ```yaml