mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
Merge branch 'main' into litellm_default_router_retries
This commit is contained in:
Krish Dholakia
GitHub
commit
4b0f73500f
55 changed files with 994 additions and 218 deletions
|
|
@ -231,13 +231,16 @@ Your OpenAI proxy server is now running on `http://127.0.0.1:4000`.
|
|||
| Docs | When to Use |
|
||||
| --- | --- |
|
||||
| [Quick Start](#quick-start) | call 100+ LLMs + Load Balancing |
|
||||
| [Deploy with Database](#deploy-with-database) | + use Virtual Keys + Track Spend |
|
||||
| [Deploy with Database](#deploy-with-database) | + use Virtual Keys + Track Spend (Note: When deploying with a database providing a `DATABASE_URL` and `LITELLM_MASTER_KEY` are required in your env ) |
|
||||
| [LiteLLM container + Redis](#litellm-container--redis) | + load balance across multiple litellm containers |
|
||||
| [LiteLLM Database container + PostgresDB + Redis](#litellm-database-container--postgresdb--redis) | + use Virtual Keys + Track Spend + load balance across multiple litellm containers |
|
||||
|
||||
## Deploy with Database
|
||||
### Docker, Kubernetes, Helm Chart
|
||||
|
||||
Requirements:
|
||||
- Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) Set `DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname>` in your env
|
||||
- Set a `LITELLM_MASTER_KEY`, this is your Proxy Admin key - you can use this to create other keys (🚨 must start with `sk-`)
|
||||
|
||||
<Tabs>
|
||||
|
||||
|
|
@ -252,6 +255,8 @@ docker pull ghcr.io/berriai/litellm-database:main-latest
|
|||
```shell
|
||||
docker run \
|
||||
-v $(pwd)/litellm_config.yaml:/app/config.yaml \
|
||||
-e LITELLM_MASTER_KEY=sk-1234 \
|
||||
-e DATABASE_URL=postgresql://<user>:<password>@<host>:<port>/<dbname> \
|
||||
-e AZURE_API_KEY=d6*********** \
|
||||
-e AZURE_API_BASE=https://openai-***********/ \
|
||||
-p 4000:4000 \
|
||||
|
|
|
|||
|
|
@ -95,7 +95,7 @@ print(response)
|
|||
- `router.image_generation()` - completion calls in OpenAI `/v1/images/generations` endpoint format
|
||||
- `router.aimage_generation()` - async image generation calls
|
||||
|
||||
### Advanced - Routing Strategies
|
||||
## Advanced - Routing Strategies
|
||||
#### Routing Strategies - Weighted Pick, Rate Limit Aware, Least Busy, Latency Based
|
||||
|
||||
Router provides 4 strategies for routing your calls across multiple deployments:
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ However, we also expose 3 public helper functions to calculate token usage acros
|
|||
```python
|
||||
from litellm import token_counter
|
||||
|
||||
messages = [{"user": "role", "content": "Hey, how's it going"}]
|
||||
messages = [{"role": "user", "content": "Hey, how's it going"}]
|
||||
print(token_counter(model="gpt-3.5-turbo", messages=messages))
|
||||
```
|
||||
|
||||
|
|
|
|||
8
litellm-js/spend-logs/package-lock.json
generated
8
litellm-js/spend-logs/package-lock.json
generated
|
|
@ -6,7 +6,7 @@
|
|||
"": {
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.9.0",
|
||||
"hono": "^4.1.5"
|
||||
"hono": "^4.2.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.17",
|
||||
|
|
@ -463,9 +463,9 @@
|
|||
}
|
||||
},
|
||||
"node_modules/hono": {
|
||||
"version": "4.1.5",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.1.5.tgz",
|
||||
"integrity": "sha512-3ChJiIoeCxvkt6vnkxJagplrt1YZg3NyNob7ssVeK2PUqEINp4q1F94HzFnvY9QE8asVmbW5kkTDlyWylfg2vg==",
|
||||
"version": "4.2.7",
|
||||
"resolved": "https://registry.npmjs.org/hono/-/hono-4.2.7.tgz",
|
||||
"integrity": "sha512-k1xHi86tJnRIVvqhFMBDGFKJ8r5O+bEsT4P59ZK59r0F300Xd910/r237inVfuT/VmE86RQQffX4OYNda6dLXw==",
|
||||
"engines": {
|
||||
"node": ">=16.0.0"
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@
|
|||
},
|
||||
"dependencies": {
|
||||
"@hono/node-server": "^1.9.0",
|
||||
"hono": "^4.1.5"
|
||||
"hono": "^4.2.7"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.17",
|
||||
|
|
|
|||
|
|
@ -84,6 +84,7 @@ class LangFuseLogger:
|
|||
print_verbose(
|
||||
f"Langfuse Logging - Enters logging function for model {kwargs}"
|
||||
)
|
||||
|
||||
litellm_params = kwargs.get("litellm_params", {})
|
||||
metadata = (
|
||||
litellm_params.get("metadata", {}) or {}
|
||||
|
|
@ -373,7 +374,11 @@ class LangFuseLogger:
|
|||
# just log `litellm-{call_type}` as the generation name
|
||||
generation_name = f"litellm-{kwargs.get('call_type', 'completion')}"
|
||||
|
||||
system_fingerprint = response_obj.get("system_fingerprint", None)
|
||||
if response_obj is not None and "system_fingerprint" in response_obj:
|
||||
system_fingerprint = response_obj.get("system_fingerprint", None)
|
||||
else:
|
||||
system_fingerprint = None
|
||||
|
||||
if system_fingerprint is not None:
|
||||
optional_params["system_fingerprint"] = system_fingerprint
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ import copy
|
|||
import traceback
|
||||
from litellm._logging import verbose_logger, verbose_proxy_logger
|
||||
import litellm
|
||||
from typing import List, Literal, Any, Union, Optional
|
||||
from typing import List, Literal, Any, Union, Optional, Dict
|
||||
from litellm.caching import DualCache
|
||||
import asyncio
|
||||
import aiohttp
|
||||
|
|
@ -37,12 +37,16 @@ class SlackAlerting:
|
|||
"budget_alerts",
|
||||
"db_exceptions",
|
||||
],
|
||||
alert_to_webhook_url: Optional[
|
||||
Dict
|
||||
] = None, # if user wants to separate alerts to diff channels
|
||||
):
|
||||
self.alerting_threshold = alerting_threshold
|
||||
self.alerting = alerting
|
||||
self.alert_types = alert_types
|
||||
self.internal_usage_cache = DualCache()
|
||||
self.async_http_handler = AsyncHTTPHandler()
|
||||
self.alert_to_webhook_url = alert_to_webhook_url
|
||||
|
||||
pass
|
||||
|
||||
|
|
@ -51,6 +55,7 @@ class SlackAlerting:
|
|||
alerting: Optional[List] = None,
|
||||
alerting_threshold: Optional[float] = None,
|
||||
alert_types: Optional[List] = None,
|
||||
alert_to_webhook_url: Optional[Dict] = None,
|
||||
):
|
||||
if alerting is not None:
|
||||
self.alerting = alerting
|
||||
|
|
@ -59,6 +64,13 @@ class SlackAlerting:
|
|||
if alert_types is not None:
|
||||
self.alert_types = alert_types
|
||||
|
||||
if alert_to_webhook_url is not None:
|
||||
# update the dict
|
||||
if self.alert_to_webhook_url is None:
|
||||
self.alert_to_webhook_url = alert_to_webhook_url
|
||||
else:
|
||||
self.alert_to_webhook_url.update(alert_to_webhook_url)
|
||||
|
||||
async def deployment_in_cooldown(self):
|
||||
pass
|
||||
|
||||
|
|
@ -140,7 +152,6 @@ class SlackAlerting:
|
|||
raise e
|
||||
|
||||
def _get_deployment_latencies_to_alert(self, metadata=None):
|
||||
|
||||
if metadata is None:
|
||||
return None
|
||||
|
||||
|
|
@ -171,8 +182,6 @@ class SlackAlerting:
|
|||
if self.alerting is None or self.alert_types is None:
|
||||
return
|
||||
|
||||
if "llm_too_slow" not in self.alert_types:
|
||||
return
|
||||
time_difference_float, model, api_base, messages = (
|
||||
self._response_taking_too_long_callback(
|
||||
kwargs=kwargs,
|
||||
|
|
@ -205,6 +214,7 @@ class SlackAlerting:
|
|||
await self.send_alert(
|
||||
message=slow_message + request_info,
|
||||
level="Low",
|
||||
alert_type="llm_too_slow",
|
||||
)
|
||||
|
||||
async def log_failure_event(self, original_exception: Exception):
|
||||
|
|
@ -241,9 +251,6 @@ class SlackAlerting:
|
|||
request_info = ""
|
||||
|
||||
if type == "hanging_request":
|
||||
# Simulate a long-running operation that could take more than 5 minutes
|
||||
if "llm_requests_hanging" not in self.alert_types:
|
||||
return
|
||||
await asyncio.sleep(
|
||||
self.alerting_threshold
|
||||
) # Set it to 5 minutes - i'd imagine this might be different for streaming, non-streaming, non-completion (embedding + img) requests
|
||||
|
|
@ -291,6 +298,7 @@ class SlackAlerting:
|
|||
await self.send_alert(
|
||||
message=alerting_message + request_info,
|
||||
level="Medium",
|
||||
alert_type="llm_requests_hanging",
|
||||
)
|
||||
|
||||
async def budget_alerts(
|
||||
|
|
@ -336,8 +344,7 @@ class SlackAlerting:
|
|||
user_info = f"\nUser ID: {user_id}\n Error {error_message}"
|
||||
message = "Failed Tracking Cost for" + user_info
|
||||
await self.send_alert(
|
||||
message=message,
|
||||
level="High",
|
||||
message=message, level="High", alert_type="budget_alerts"
|
||||
)
|
||||
return
|
||||
elif type == "projected_limit_exceeded" and user_info is not None:
|
||||
|
|
@ -353,8 +360,7 @@ class SlackAlerting:
|
|||
"""
|
||||
message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` {user_info["key_alias"]} \n`Expected Day of Error`: {user_info["projected_exceeded_date"]} \n`Current Spend`: {user_current_spend} \n`Projected Spend at end of month`: {user_info["projected_spend"]} \n`Soft Limit`: {user_max_budget}"""
|
||||
await self.send_alert(
|
||||
message=message,
|
||||
level="High",
|
||||
message=message, level="High", alert_type="budget_alerts"
|
||||
)
|
||||
return
|
||||
else:
|
||||
|
|
@ -382,8 +388,7 @@ class SlackAlerting:
|
|||
result = await _cache.async_get_cache(key=message)
|
||||
if result is None:
|
||||
await self.send_alert(
|
||||
message=message,
|
||||
level="High",
|
||||
message=message, level="High", alert_type="budget_alerts"
|
||||
)
|
||||
await _cache.async_set_cache(key=message, value="SENT", ttl=2419200)
|
||||
return
|
||||
|
|
@ -395,8 +400,7 @@ class SlackAlerting:
|
|||
result = await _cache.async_get_cache(key=cache_key)
|
||||
if result is None:
|
||||
await self.send_alert(
|
||||
message=message,
|
||||
level="Medium",
|
||||
message=message, level="Medium", alert_type="budget_alerts"
|
||||
)
|
||||
|
||||
await _cache.async_set_cache(key=cache_key, value="SENT", ttl=2419200)
|
||||
|
|
@ -409,15 +413,25 @@ class SlackAlerting:
|
|||
result = await _cache.async_get_cache(key=message)
|
||||
if result is None:
|
||||
await self.send_alert(
|
||||
message=message,
|
||||
level="Low",
|
||||
message=message, level="Low", alert_type="budget_alerts"
|
||||
)
|
||||
await _cache.async_set_cache(key=message, value="SENT", ttl=2419200)
|
||||
return
|
||||
|
||||
return
|
||||
|
||||
async def send_alert(self, message: str, level: Literal["Low", "Medium", "High"]):
|
||||
async def send_alert(
|
||||
self,
|
||||
message: str,
|
||||
level: Literal["Low", "Medium", "High"],
|
||||
alert_type: Literal[
|
||||
"llm_exceptions",
|
||||
"llm_too_slow",
|
||||
"llm_requests_hanging",
|
||||
"budget_alerts",
|
||||
"db_exceptions",
|
||||
],
|
||||
):
|
||||
"""
|
||||
Alerting based on thresholds: - https://github.com/BerriAI/litellm/issues/1298
|
||||
|
||||
|
|
@ -432,12 +446,6 @@ class SlackAlerting:
|
|||
level: str - Low|Medium|High - if calls might fail (Medium) or are failing (High); Currently, no alerts would be 'Low'.
|
||||
message: str - what is the alert about
|
||||
"""
|
||||
print(
|
||||
"inside send alert for slack, message: ",
|
||||
message,
|
||||
"self.alerting: ",
|
||||
self.alerting,
|
||||
)
|
||||
if self.alerting is None:
|
||||
return
|
||||
|
||||
|
|
@ -453,7 +461,15 @@ class SlackAlerting:
|
|||
if _proxy_base_url is not None:
|
||||
formatted_message += f"\n\nProxy URL: `{_proxy_base_url}`"
|
||||
|
||||
slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None)
|
||||
# check if we find the slack webhook url in self.alert_to_webhook_url
|
||||
if (
|
||||
self.alert_to_webhook_url is not None
|
||||
and alert_type in self.alert_to_webhook_url
|
||||
):
|
||||
slack_webhook_url = self.alert_to_webhook_url[alert_type]
|
||||
else:
|
||||
slack_webhook_url = os.getenv("SLACK_WEBHOOK_URL", None)
|
||||
|
||||
if slack_webhook_url is None:
|
||||
raise Exception("Missing SLACK_WEBHOOK_URL from environment")
|
||||
payload = {"text": formatted_message}
|
||||
|
|
|
|||
|
|
@ -653,6 +653,10 @@ def convert_messages_to_prompt(model, messages, provider, custom_prompt_dict):
|
|||
prompt = prompt_factory(
|
||||
model=model, messages=messages, custom_llm_provider="bedrock"
|
||||
)
|
||||
elif provider == "meta":
|
||||
prompt = prompt_factory(
|
||||
model=model, messages=messages, custom_llm_provider="bedrock"
|
||||
)
|
||||
else:
|
||||
prompt = ""
|
||||
for message in messages:
|
||||
|
|
|
|||
|
|
@ -1346,6 +1346,13 @@ def prompt_factory(
|
|||
return anthropic_pt(messages=messages)
|
||||
elif "mistral." in model:
|
||||
return mistral_instruct_pt(messages=messages)
|
||||
elif "llama2" in model and "chat" in model:
|
||||
return llama_2_chat_pt(messages=messages)
|
||||
elif "llama3" in model and "instruct" in model:
|
||||
return hf_chat_template(
|
||||
model="meta-llama/Meta-Llama-3-8B-Instruct",
|
||||
messages=messages,
|
||||
)
|
||||
elif custom_llm_provider == "perplexity":
|
||||
for message in messages:
|
||||
message.pop("name", None)
|
||||
|
|
|
|||
|
|
@ -143,7 +143,9 @@ class VertexAIConfig:
|
|||
optional_params["temperature"] = value
|
||||
if param == "top_p":
|
||||
optional_params["top_p"] = value
|
||||
if param == "stream":
|
||||
if (
|
||||
param == "stream" and value == True
|
||||
): # sending stream = False, can cause it to get passed unchecked and raise issues
|
||||
optional_params["stream"] = value
|
||||
if param == "n":
|
||||
optional_params["candidate_count"] = value
|
||||
|
|
@ -541,8 +543,9 @@ def completion(
|
|||
tools = optional_params.pop("tools", None)
|
||||
prompt, images = _gemini_vision_convert_messages(messages=messages)
|
||||
content = [prompt] + images
|
||||
if "stream" in optional_params and optional_params["stream"] == True:
|
||||
stream = optional_params.pop("stream")
|
||||
stream = optional_params.pop("stream", False)
|
||||
if stream == True:
|
||||
|
||||
request_str += f"response = llm_model.generate_content({content}, generation_config=GenerationConfig(**{optional_params}), safety_settings={safety_settings}, stream={stream})\n"
|
||||
logging_obj.pre_call(
|
||||
input=prompt,
|
||||
|
|
@ -820,6 +823,7 @@ async def async_completion(
|
|||
print_verbose("\nMaking VertexAI Gemini Pro/Vision Call")
|
||||
print_verbose(f"\nProcessing input messages = {messages}")
|
||||
tools = optional_params.pop("tools", None)
|
||||
stream = optional_params.pop("stream", False)
|
||||
|
||||
prompt, images = _gemini_vision_convert_messages(messages=messages)
|
||||
content = [prompt] + images
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ import dotenv, traceback, random, asyncio, time, contextvars
|
|||
from copy import deepcopy
|
||||
import httpx
|
||||
import litellm
|
||||
|
||||
from ._logging import verbose_logger
|
||||
from litellm import ( # type: ignore
|
||||
client,
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/60d9f441227ccc7e.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
|
||||
!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/5e699db73bf6f8c2.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-50c1dadc6557c101.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-dafd44dfa2da140c.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e49705773ae41779.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-50c1dadc6557c101.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/60d9f441227ccc7e.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[82332,[\"127\",\"static/chunks/127-efd0436630e294eb.js\",\"931\",\"static/chunks/app/page-525d83925fd5350b.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/60d9f441227ccc7e.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"Csz8BqWx6JEoKsgLqCeCt\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-ccae12a25017afa5.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-dafd44dfa2da140c.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e49705773ae41779.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-ccae12a25017afa5.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/5e699db73bf6f8c2.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[27125,[\"447\",\"static/chunks/447-9f8d32190ff7d16d.js\",\"931\",\"static/chunks/app/page-781ca5f151d78d1d.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/5e699db73bf6f8c2.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"PtTtxXIYvdjQsvRgdITlk\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[77831,[],""]
|
||||
3:I[82332,["127","static/chunks/127-efd0436630e294eb.js","931","static/chunks/app/page-525d83925fd5350b.js"],""]
|
||||
3:I[27125,["447","static/chunks/447-9f8d32190ff7d16d.js","931","static/chunks/app/page-781ca5f151d78d1d.js"],""]
|
||||
4:I[5613,[],""]
|
||||
5:I[31778,[],""]
|
||||
0:["Csz8BqWx6JEoKsgLqCeCt",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/60d9f441227ccc7e.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
0:["PtTtxXIYvdjQsvRgdITlk",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/5e699db73bf6f8c2.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -6,3 +6,4 @@ model_list:
|
|||
model_name: fake-openai-endpoint
|
||||
router_settings:
|
||||
num_retries: 0
|
||||
|
||||
|
|
|
|||
|
|
@ -720,6 +720,10 @@ class ConfigGeneralSettings(LiteLLMBase):
|
|||
None,
|
||||
description="List of alerting types. By default it is all alerts",
|
||||
)
|
||||
alert_to_webhook_url: Optional[Dict] = Field(
|
||||
None,
|
||||
description="Mapping of alert type to webhook url. e.g. `alert_to_webhook_url: {'budget_alerts': 'https://hooks.slack.com/services/T00000000/B00000000/XXXXXXXXXXXXXXXXXXXXXXXX'}`",
|
||||
)
|
||||
|
||||
alerting_threshold: Optional[int] = Field(
|
||||
None,
|
||||
|
|
|
|||
|
|
@ -2632,9 +2632,17 @@ class ProxyConfig:
|
|||
if "alert_types" in _general_settings:
|
||||
general_settings["alert_types"] = _general_settings["alert_types"]
|
||||
proxy_logging_obj.alert_types = general_settings["alert_types"]
|
||||
proxy_logging_obj.slack_alerting_instance.alert_types = general_settings[
|
||||
"alert_types"
|
||||
proxy_logging_obj.slack_alerting_instance.update_values(
|
||||
alert_types=general_settings["alert_types"]
|
||||
)
|
||||
|
||||
if "alert_to_webhook_url" in _general_settings:
|
||||
general_settings["alert_to_webhook_url"] = _general_settings[
|
||||
"alert_to_webhook_url"
|
||||
]
|
||||
proxy_logging_obj.slack_alerting_instance.update_values(
|
||||
alert_to_webhook_url=general_settings["alert_to_webhook_url"]
|
||||
)
|
||||
|
||||
# router settings
|
||||
if llm_router is not None and prisma_client is not None:
|
||||
|
|
@ -3655,6 +3663,17 @@ async def chat_completion(
|
|||
if data["model"] in litellm.model_alias_map:
|
||||
data["model"] = litellm.model_alias_map[data["model"]]
|
||||
|
||||
## LOGGING OBJECT ## - initialize logging object for logging success/failure events for call
|
||||
data["litellm_call_id"] = str(uuid.uuid4())
|
||||
logging_obj, data = litellm.utils.function_setup(
|
||||
original_function="acompletion",
|
||||
rules_obj=litellm.utils.Rules(),
|
||||
start_time=datetime.now(),
|
||||
**data,
|
||||
)
|
||||
|
||||
data["litellm_logging_obj"] = logging_obj
|
||||
|
||||
### CALL HOOKS ### - modify incoming data before calling the model
|
||||
data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict, data=data, call_type="completion"
|
||||
|
|
@ -8592,6 +8611,7 @@ async def get_config():
|
|||
|
||||
# Check if slack alerting is on
|
||||
_alerting = _general_settings.get("alerting", [])
|
||||
alerting_data = []
|
||||
if "slack" in _alerting:
|
||||
_slack_vars = [
|
||||
"SLACK_WEBHOOK_URL",
|
||||
|
|
@ -8600,7 +8620,8 @@ async def get_config():
|
|||
for _var in _slack_vars:
|
||||
env_variable = environment_variables.get(_var, None)
|
||||
if env_variable is None:
|
||||
_slack_env_vars[_var] = None
|
||||
_value = os.getenv("SLACK_WEBHOOK_URL", None)
|
||||
_slack_env_vars[_var] = _value
|
||||
else:
|
||||
# decode + decrypt the value
|
||||
decoded_b64 = base64.b64decode(env_variable)
|
||||
|
|
@ -8613,19 +8634,23 @@ async def get_config():
|
|||
_all_alert_types = (
|
||||
proxy_logging_obj.slack_alerting_instance._all_possible_alert_types()
|
||||
)
|
||||
_data_to_return.append(
|
||||
_alerts_to_webhook = (
|
||||
proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url
|
||||
)
|
||||
alerting_data.append(
|
||||
{
|
||||
"name": "slack",
|
||||
"variables": _slack_env_vars,
|
||||
"alerting_types": _alerting_types,
|
||||
"all_alert_types": _all_alert_types,
|
||||
"active_alerts": _alerting_types,
|
||||
"alerts_to_webhook": _alerts_to_webhook,
|
||||
}
|
||||
)
|
||||
|
||||
_router_settings = llm_router.get_settings()
|
||||
return {
|
||||
"status": "success",
|
||||
"data": _data_to_return,
|
||||
"callbacks": _data_to_return,
|
||||
"alerts": alerting_data,
|
||||
"router_settings": _router_settings,
|
||||
}
|
||||
except Exception as e:
|
||||
|
|
@ -8742,8 +8767,51 @@ async def health_services_endpoint(
|
|||
}
|
||||
|
||||
if "slack" in general_settings.get("alerting", []):
|
||||
test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
|
||||
await proxy_logging_obj.alerting_handler(message=test_message, level="Low")
|
||||
# test_message = f"""\n🚨 `ProjectedLimitExceededError` 💸\n\n`Key Alias:` litellm-ui-test-alert \n`Expected Day of Error`: 28th March \n`Current Spend`: $100.00 \n`Projected Spend at end of month`: $1000.00 \n`Soft Limit`: $700"""
|
||||
# check if user has opted into unique_alert_webhooks
|
||||
if (
|
||||
proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url
|
||||
is not None
|
||||
):
|
||||
for (
|
||||
alert_type
|
||||
) in proxy_logging_obj.slack_alerting_instance.alert_to_webhook_url:
|
||||
"""
|
||||
"llm_exceptions",
|
||||
"llm_too_slow",
|
||||
"llm_requests_hanging",
|
||||
"budget_alerts",
|
||||
"db_exceptions",
|
||||
"""
|
||||
# only test alert if it's in active alert types
|
||||
if (
|
||||
proxy_logging_obj.slack_alerting_instance.alert_types
|
||||
is not None
|
||||
and alert_type
|
||||
not in proxy_logging_obj.slack_alerting_instance.alert_types
|
||||
):
|
||||
continue
|
||||
test_message = "default test message"
|
||||
if alert_type == "llm_exceptions":
|
||||
test_message = f"LLM Exception test alert"
|
||||
elif alert_type == "llm_too_slow":
|
||||
test_message = f"LLM Too Slow test alert"
|
||||
elif alert_type == "llm_requests_hanging":
|
||||
test_message = f"LLM Requests Hanging test alert"
|
||||
elif alert_type == "budget_alerts":
|
||||
test_message = f"Budget Alert test alert"
|
||||
elif alert_type == "db_exceptions":
|
||||
test_message = f"DB Exception test alert"
|
||||
|
||||
await proxy_logging_obj.alerting_handler(
|
||||
message=test_message, level="Low", alert_type=alert_type
|
||||
)
|
||||
else:
|
||||
await proxy_logging_obj.alerting_handler(
|
||||
message="This is a test slack alert message",
|
||||
level="Low",
|
||||
alert_type="budget_alerts",
|
||||
)
|
||||
return {
|
||||
"status": "success",
|
||||
"message": "Mock Slack Alert sent, verify Slack Alert Received on your channel",
|
||||
|
|
@ -8761,7 +8829,7 @@ async def health_services_endpoint(
|
|||
message=getattr(e, "detail", f"Authentication Error({str(e)})"),
|
||||
type="auth_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
code=getattr(e, "status_code", status.HTTP_401_UNAUTHORIZED),
|
||||
code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
|
||||
)
|
||||
elif isinstance(e, ProxyException):
|
||||
raise e
|
||||
|
|
@ -8769,7 +8837,7 @@ async def health_services_endpoint(
|
|||
message="Authentication Error, " + str(e),
|
||||
type="auth_error",
|
||||
param=getattr(e, "param", "None"),
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
from typing import Optional, List, Any, Literal, Union
|
||||
import os, subprocess, hashlib, importlib, asyncio, copy, json, aiohttp, httpx, time
|
||||
import litellm, backoff
|
||||
import litellm, backoff, traceback
|
||||
from litellm.proxy._types import (
|
||||
UserAPIKeyAuth,
|
||||
DynamoDBArgs,
|
||||
|
|
@ -199,6 +199,33 @@ class ProxyLogging:
|
|||
print_verbose(f"final data being sent to {call_type} call: {data}")
|
||||
return data
|
||||
except Exception as e:
|
||||
if "litellm_logging_obj" in data:
|
||||
logging_obj: litellm.utils.Logging = data["litellm_logging_obj"]
|
||||
|
||||
## ASYNC FAILURE HANDLER ##
|
||||
error_message = ""
|
||||
if isinstance(e, HTTPException):
|
||||
if isinstance(e.detail, str):
|
||||
error_message = e.detail
|
||||
elif isinstance(e.detail, dict):
|
||||
error_message = json.dumps(e.detail)
|
||||
else:
|
||||
error_message = str(e)
|
||||
else:
|
||||
error_message = str(e)
|
||||
error_raised = Exception(f"{error_message}")
|
||||
await logging_obj.async_failure_handler(
|
||||
exception=error_raised,
|
||||
traceback_exception=traceback.format_exc(),
|
||||
)
|
||||
|
||||
## SYNC FAILURE HANDLER ##
|
||||
try:
|
||||
logging_obj.failure_handler(
|
||||
error_raised, traceback.format_exc()
|
||||
) # DO NOT MAKE THREADED - router retry fallback relies on this!
|
||||
except Exception as error_val:
|
||||
pass
|
||||
raise e
|
||||
|
||||
async def during_call_hook(
|
||||
|
|
@ -256,7 +283,16 @@ class ProxyLogging:
|
|||
)
|
||||
|
||||
async def alerting_handler(
|
||||
self, message: str, level: Literal["Low", "Medium", "High"]
|
||||
self,
|
||||
message: str,
|
||||
level: Literal["Low", "Medium", "High"],
|
||||
alert_type: Literal[
|
||||
"llm_exceptions",
|
||||
"llm_too_slow",
|
||||
"llm_requests_hanging",
|
||||
"budget_alerts",
|
||||
"db_exceptions",
|
||||
],
|
||||
):
|
||||
"""
|
||||
Alerting based on thresholds: - https://github.com/BerriAI/litellm/issues/1298
|
||||
|
|
@ -289,7 +325,7 @@ class ProxyLogging:
|
|||
for client in self.alerting:
|
||||
if client == "slack":
|
||||
await self.slack_alerting_instance.send_alert(
|
||||
message=message, level=level
|
||||
message=message, level=level, alert_type=alert_type
|
||||
)
|
||||
elif client == "sentry":
|
||||
if litellm.utils.sentry_sdk_instance is not None:
|
||||
|
|
@ -323,6 +359,7 @@ class ProxyLogging:
|
|||
self.alerting_handler(
|
||||
message=f"DB read/write call failed: {error_message}",
|
||||
level="High",
|
||||
alert_type="db_exceptions",
|
||||
)
|
||||
)
|
||||
|
||||
|
|
@ -354,7 +391,9 @@ class ProxyLogging:
|
|||
return
|
||||
asyncio.create_task(
|
||||
self.alerting_handler(
|
||||
message=f"LLM API call failed: {str(original_exception)}", level="High"
|
||||
message=f"LLM API call failed: {str(original_exception)}",
|
||||
level="High",
|
||||
alert_type="llm_exceptions",
|
||||
)
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -345,6 +345,21 @@ class LowestLatencyLoggingHandler(CustomLogger):
|
|||
if isinstance(_call_latency, float):
|
||||
total += _call_latency
|
||||
item_latency = total / len(item_latency)
|
||||
|
||||
# -------------- #
|
||||
# Debugging Logic
|
||||
# -------------- #
|
||||
# We use _latency_per_deployment to log to langfuse, slack - this is not used to make a decision on routing
|
||||
# this helps a user to debug why the router picked a specfic deployment #
|
||||
_deployment_api_base = _deployment.get("litellm_params", {}).get(
|
||||
"api_base", ""
|
||||
)
|
||||
if _deployment_api_base is not None:
|
||||
_latency_per_deployment[_deployment_api_base] = item_latency
|
||||
# -------------- #
|
||||
# End of Debugging Logic
|
||||
# -------------- #
|
||||
|
||||
if item_latency == 0:
|
||||
deployment = _deployment
|
||||
break
|
||||
|
|
@ -356,12 +371,6 @@ class LowestLatencyLoggingHandler(CustomLogger):
|
|||
elif item_latency < lowest_latency:
|
||||
lowest_latency = item_latency
|
||||
deployment = _deployment
|
||||
|
||||
# _latency_per_deployment is used for debuggig
|
||||
_deployment_api_base = _deployment.get("litellm_params", {}).get(
|
||||
"api_base", ""
|
||||
)
|
||||
_latency_per_deployment[_deployment_api_base] = item_latency
|
||||
if request_kwargs is not None and "metadata" in request_kwargs:
|
||||
request_kwargs["metadata"][
|
||||
"_latency_per_deployment"
|
||||
|
|
|
|||
|
|
@ -68,6 +68,7 @@ async def test_get_api_base():
|
|||
await _pl.alerting_handler(
|
||||
message=slow_message + request_info,
|
||||
level="Low",
|
||||
alert_type="llm_too_slow",
|
||||
)
|
||||
print("passed test_get_api_base")
|
||||
|
||||
|
|
|
|||
|
|
@ -636,7 +636,10 @@ def test_gemini_pro_function_calling():
|
|||
# gemini_pro_function_calling()
|
||||
|
||||
|
||||
def test_gemini_pro_function_calling_streaming():
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("sync_mode", [False, True])
|
||||
@pytest.mark.asyncio
|
||||
async def test_gemini_pro_function_calling_streaming(stream, sync_mode):
|
||||
load_vertex_ai_credentials()
|
||||
litellm.set_verbose = True
|
||||
tools = [
|
||||
|
|
@ -665,19 +668,41 @@ def test_gemini_pro_function_calling_streaming():
|
|||
"content": "What's the weather like in Boston today in fahrenheit?",
|
||||
}
|
||||
]
|
||||
optional_params = {
|
||||
"tools": tools,
|
||||
"tool_choice": "auto",
|
||||
"n": 1,
|
||||
"stream": stream,
|
||||
"temperature": 0.1,
|
||||
}
|
||||
try:
|
||||
completion = litellm.completion(
|
||||
model="gemini-pro",
|
||||
messages=messages,
|
||||
tools=tools,
|
||||
tool_choice="auto",
|
||||
stream=True,
|
||||
)
|
||||
print(f"completion: {completion}")
|
||||
# assert completion.choices[0].message.content is None
|
||||
# assert len(completion.choices[0].message.tool_calls) == 1
|
||||
for chunk in completion:
|
||||
print(f"chunk: {chunk}")
|
||||
if sync_mode == True:
|
||||
response = litellm.completion(
|
||||
model="gemini-pro", messages=messages, **optional_params
|
||||
)
|
||||
print(f"completion: {response}")
|
||||
|
||||
if stream == True:
|
||||
# assert completion.choices[0].message.content is None
|
||||
# assert len(completion.choices[0].message.tool_calls) == 1
|
||||
for chunk in response:
|
||||
assert isinstance(chunk, litellm.ModelResponse)
|
||||
else:
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
else:
|
||||
response = await litellm.acompletion(
|
||||
model="gemini-pro", messages=messages, **optional_params
|
||||
)
|
||||
print(f"completion: {response}")
|
||||
|
||||
if stream == True:
|
||||
# assert completion.choices[0].message.content is None
|
||||
# assert len(completion.choices[0].message.tool_calls) == 1
|
||||
async for chunk in response:
|
||||
print(f"chunk: {chunk}")
|
||||
assert isinstance(chunk, litellm.ModelResponse)
|
||||
else:
|
||||
assert isinstance(response, litellm.ModelResponse)
|
||||
except litellm.APIError as e:
|
||||
pass
|
||||
except litellm.RateLimitError as e:
|
||||
|
|
|
|||
|
|
@ -1291,6 +1291,7 @@ def test_completion_logprobs_stream():
|
|||
for chunk in response:
|
||||
# check if atleast one chunk has log probs
|
||||
print(chunk)
|
||||
print(f"chunk.choices[0]: {chunk.choices[0]}")
|
||||
if "logprobs" in chunk.choices[0]:
|
||||
# assert we got a valid logprob in the choices
|
||||
assert len(chunk.choices[0].logprobs.content[0].top_logprobs) == 3
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ def test_empty_content():
|
|||
pass
|
||||
|
||||
function_setup(
|
||||
original_function=completion,
|
||||
original_function="completion",
|
||||
rules_obj=rules_obj,
|
||||
start_time=datetime.now(),
|
||||
messages=[],
|
||||
|
|
|
|||
|
|
@ -2446,6 +2446,34 @@ class ModelResponseIterator:
|
|||
return self.model_response
|
||||
|
||||
|
||||
class ModelResponseListIterator:
|
||||
def __init__(self, model_responses):
|
||||
self.model_responses = model_responses
|
||||
self.index = 0
|
||||
|
||||
# Sync iterator
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __next__(self):
|
||||
if self.index >= len(self.model_responses):
|
||||
raise StopIteration
|
||||
model_response = self.model_responses[self.index]
|
||||
self.index += 1
|
||||
return model_response
|
||||
|
||||
# Async iterator
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
async def __anext__(self):
|
||||
if self.index >= len(self.model_responses):
|
||||
raise StopAsyncIteration
|
||||
model_response = self.model_responses[self.index]
|
||||
self.index += 1
|
||||
return model_response
|
||||
|
||||
|
||||
def test_unit_test_custom_stream_wrapper():
|
||||
"""
|
||||
Test if last streaming chunk ends with '?', if the message repeats itself.
|
||||
|
|
@ -2486,3 +2514,268 @@ def test_unit_test_custom_stream_wrapper():
|
|||
if "How are you?" in chunk.choices[0].delta.content:
|
||||
freq += 1
|
||||
assert freq == 1
|
||||
|
||||
|
||||
def test_aamazing_unit_test_custom_stream_wrapper_n():
|
||||
"""
|
||||
Test if the translated output maps exactly to the received openai input
|
||||
|
||||
Relevant issue: https://github.com/BerriAI/litellm/issues/3276
|
||||
"""
|
||||
chunks = [
|
||||
{
|
||||
"id": "chatcmpl-9HzZIMCtVq7CbTmdwEZrktiTeoiYe",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1714075272,
|
||||
"model": "gpt-4-0613",
|
||||
"system_fingerprint": None,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "It"},
|
||||
"logprobs": {
|
||||
"content": [
|
||||
{
|
||||
"token": "It",
|
||||
"logprob": -1.5952516,
|
||||
"bytes": [73, 116],
|
||||
"top_logprobs": [
|
||||
{
|
||||
"token": "Brown",
|
||||
"logprob": -0.7358765,
|
||||
"bytes": [66, 114, 111, 119, 110],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-9HzZIMCtVq7CbTmdwEZrktiTeoiYe",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1714075272,
|
||||
"model": "gpt-4-0613",
|
||||
"system_fingerprint": None,
|
||||
"choices": [
|
||||
{
|
||||
"index": 1,
|
||||
"delta": {"content": "Brown"},
|
||||
"logprobs": {
|
||||
"content": [
|
||||
{
|
||||
"token": "Brown",
|
||||
"logprob": -0.7358765,
|
||||
"bytes": [66, 114, 111, 119, 110],
|
||||
"top_logprobs": [
|
||||
{
|
||||
"token": "Brown",
|
||||
"logprob": -0.7358765,
|
||||
"bytes": [66, 114, 111, 119, 110],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-9HzZIMCtVq7CbTmdwEZrktiTeoiYe",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1714075272,
|
||||
"model": "gpt-4-0613",
|
||||
"system_fingerprint": None,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "'s"},
|
||||
"logprobs": {
|
||||
"content": [
|
||||
{
|
||||
"token": "'s",
|
||||
"logprob": -0.006786893,
|
||||
"bytes": [39, 115],
|
||||
"top_logprobs": [
|
||||
{
|
||||
"token": "'s",
|
||||
"logprob": -0.006786893,
|
||||
"bytes": [39, 115],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-9HzZIMCtVq7CbTmdwEZrktiTeoiYe",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1714075272,
|
||||
"model": "gpt-4-0613",
|
||||
"system_fingerprint": None,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": " impossible"},
|
||||
"logprobs": {
|
||||
"content": [
|
||||
{
|
||||
"token": " impossible",
|
||||
"logprob": -0.06528423,
|
||||
"bytes": [
|
||||
32,
|
||||
105,
|
||||
109,
|
||||
112,
|
||||
111,
|
||||
115,
|
||||
115,
|
||||
105,
|
||||
98,
|
||||
108,
|
||||
101,
|
||||
],
|
||||
"top_logprobs": [
|
||||
{
|
||||
"token": " impossible",
|
||||
"logprob": -0.06528423,
|
||||
"bytes": [
|
||||
32,
|
||||
105,
|
||||
109,
|
||||
112,
|
||||
111,
|
||||
115,
|
||||
115,
|
||||
105,
|
||||
98,
|
||||
108,
|
||||
101,
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-9HzZIMCtVq7CbTmdwEZrktiTeoiYe",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1714075272,
|
||||
"model": "gpt-4-0613",
|
||||
"system_fingerprint": None,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"delta": {"content": "—even"},
|
||||
"logprobs": {
|
||||
"content": [
|
||||
{
|
||||
"token": "—even",
|
||||
"logprob": -9999.0,
|
||||
"bytes": [226, 128, 148, 101, 118, 101, 110],
|
||||
"top_logprobs": [
|
||||
{
|
||||
"token": " to",
|
||||
"logprob": -0.12302828,
|
||||
"bytes": [32, 116, 111],
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"finish_reason": None,
|
||||
}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-9HzZIMCtVq7CbTmdwEZrktiTeoiYe",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1714075272,
|
||||
"model": "gpt-4-0613",
|
||||
"system_fingerprint": None,
|
||||
"choices": [
|
||||
{"index": 0, "delta": {}, "logprobs": None, "finish_reason": "length"}
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "chatcmpl-9HzZIMCtVq7CbTmdwEZrktiTeoiYe",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1714075272,
|
||||
"model": "gpt-4-0613",
|
||||
"system_fingerprint": None,
|
||||
"choices": [
|
||||
{"index": 1, "delta": {}, "logprobs": None, "finish_reason": "stop"}
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
litellm.set_verbose = True
|
||||
|
||||
chunk_list = []
|
||||
for chunk in chunks:
|
||||
new_chunk = litellm.ModelResponse(stream=True, id=chunk["id"])
|
||||
if "choices" in chunk and isinstance(chunk["choices"], list):
|
||||
print("INSIDE CHUNK CHOICES!")
|
||||
new_choices = []
|
||||
for choice in chunk["choices"]:
|
||||
if isinstance(choice, litellm.utils.StreamingChoices):
|
||||
_new_choice = choice
|
||||
elif isinstance(choice, dict):
|
||||
_new_choice = litellm.utils.StreamingChoices(**choice)
|
||||
new_choices.append(_new_choice)
|
||||
new_chunk.choices = new_choices
|
||||
chunk_list.append(new_chunk)
|
||||
|
||||
completion_stream = ModelResponseListIterator(model_responses=chunk_list)
|
||||
|
||||
response = litellm.CustomStreamWrapper(
|
||||
completion_stream=completion_stream,
|
||||
model="gpt-4-0613",
|
||||
custom_llm_provider="cached_response",
|
||||
logging_obj=litellm.Logging(
|
||||
model="gpt-4-0613",
|
||||
messages=[{"role": "user", "content": "Hey"}],
|
||||
stream=True,
|
||||
call_type="completion",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="12345",
|
||||
function_id="1245",
|
||||
),
|
||||
)
|
||||
|
||||
for idx, chunk in enumerate(response):
|
||||
chunk_dict = {}
|
||||
try:
|
||||
chunk_dict = chunk.model_dump(exclude_none=True)
|
||||
except:
|
||||
chunk_dict = chunk.dict(exclude_none=True)
|
||||
|
||||
chunk_dict.pop("created")
|
||||
chunks[idx].pop("created")
|
||||
if chunks[idx]["system_fingerprint"] is None:
|
||||
chunks[idx].pop("system_fingerprint", None)
|
||||
if idx == 0:
|
||||
for choice in chunk_dict["choices"]:
|
||||
if "role" in choice["delta"]:
|
||||
choice["delta"].pop("role")
|
||||
|
||||
for choice in chunks[idx]["choices"]:
|
||||
# ignore finish reason None - since our pydantic object is set to exclude_none = true
|
||||
if "finish_reason" in choice and choice["finish_reason"] is None:
|
||||
choice.pop("finish_reason")
|
||||
if "logprobs" in choice and choice["logprobs"] is None:
|
||||
choice.pop("logprobs")
|
||||
|
||||
assert (
|
||||
chunk_dict == chunks[idx]
|
||||
), f"idx={idx} translated chunk = {chunk_dict} != openai chunk = {chunks[idx]}"
|
||||
|
|
|
|||
130
litellm/utils.py
130
litellm/utils.py
|
|
@ -19,6 +19,7 @@ from functools import wraps
|
|||
import datetime, time
|
||||
import tiktoken
|
||||
import uuid
|
||||
from pydantic import BaseModel
|
||||
import aiohttp
|
||||
import textwrap
|
||||
import logging
|
||||
|
|
@ -219,6 +220,61 @@ def map_finish_reason(
|
|||
return finish_reason
|
||||
|
||||
|
||||
class TopLogprob(OpenAIObject):
|
||||
token: str
|
||||
"""The token."""
|
||||
|
||||
bytes: Optional[List[int]] = None
|
||||
"""A list of integers representing the UTF-8 bytes representation of the token.
|
||||
|
||||
Useful in instances where characters are represented by multiple tokens and
|
||||
their byte representations must be combined to generate the correct text
|
||||
representation. Can be `null` if there is no bytes representation for the token.
|
||||
"""
|
||||
|
||||
logprob: float
|
||||
"""The log probability of this token, if it is within the top 20 most likely
|
||||
tokens.
|
||||
|
||||
Otherwise, the value `-9999.0` is used to signify that the token is very
|
||||
unlikely.
|
||||
"""
|
||||
|
||||
|
||||
class ChatCompletionTokenLogprob(OpenAIObject):
|
||||
token: str
|
||||
"""The token."""
|
||||
|
||||
bytes: Optional[List[int]] = None
|
||||
"""A list of integers representing the UTF-8 bytes representation of the token.
|
||||
|
||||
Useful in instances where characters are represented by multiple tokens and
|
||||
their byte representations must be combined to generate the correct text
|
||||
representation. Can be `null` if there is no bytes representation for the token.
|
||||
"""
|
||||
|
||||
logprob: float
|
||||
"""The log probability of this token, if it is within the top 20 most likely
|
||||
tokens.
|
||||
|
||||
Otherwise, the value `-9999.0` is used to signify that the token is very
|
||||
unlikely.
|
||||
"""
|
||||
|
||||
top_logprobs: List[TopLogprob]
|
||||
"""List of the most likely tokens and their log probability, at this token
|
||||
position.
|
||||
|
||||
In rare cases, there may be fewer than the number of requested `top_logprobs`
|
||||
returned.
|
||||
"""
|
||||
|
||||
|
||||
class ChoiceLogprobs(OpenAIObject):
|
||||
content: Optional[List[ChatCompletionTokenLogprob]] = None
|
||||
"""A list of message content tokens with log probability information."""
|
||||
|
||||
|
||||
class FunctionCall(OpenAIObject):
|
||||
arguments: str
|
||||
name: Optional[str] = None
|
||||
|
|
@ -329,7 +385,7 @@ class Message(OpenAIObject):
|
|||
self.tool_calls.append(ChatCompletionMessageToolCall(**tool_call))
|
||||
|
||||
if logprobs is not None:
|
||||
self._logprobs = logprobs
|
||||
self._logprobs = ChoiceLogprobs(**logprobs)
|
||||
|
||||
def get(self, key, default=None):
|
||||
# Custom .get() method to access attributes with a default value if the attribute doesn't exist
|
||||
|
|
@ -353,11 +409,17 @@ class Message(OpenAIObject):
|
|||
|
||||
class Delta(OpenAIObject):
|
||||
def __init__(
|
||||
self, content=None, role=None, function_call=None, tool_calls=None, **params
|
||||
self,
|
||||
content=None,
|
||||
role=None,
|
||||
function_call=None,
|
||||
tool_calls=None,
|
||||
**params,
|
||||
):
|
||||
super(Delta, self).__init__(**params)
|
||||
self.content = content
|
||||
self.role = role
|
||||
|
||||
if function_call is not None and isinstance(function_call, dict):
|
||||
self.function_call = FunctionCall(**function_call)
|
||||
else:
|
||||
|
|
@ -489,7 +551,11 @@ class StreamingChoices(OpenAIObject):
|
|||
self.delta = Delta()
|
||||
if enhancements is not None:
|
||||
self.enhancements = enhancements
|
||||
self.logprobs = logprobs
|
||||
|
||||
if logprobs is not None and isinstance(logprobs, dict):
|
||||
self.logprobs = ChoiceLogprobs(**logprobs)
|
||||
else:
|
||||
self.logprobs = logprobs # type: ignore
|
||||
|
||||
def __contains__(self, key):
|
||||
# Define custom behavior for the 'in' operator
|
||||
|
|
@ -2433,7 +2499,7 @@ class Rules:
|
|||
####### CLIENT ###################
|
||||
# make it easy to log if completion/embedding runs succeeded or failed + see what happened | Non-Blocking
|
||||
def function_setup(
|
||||
original_function, rules_obj, start_time, *args, **kwargs
|
||||
original_function: str, rules_obj, start_time, *args, **kwargs
|
||||
): # just run once to check if user wants to send their data anywhere - PostHog/Sentry/Slack/etc.
|
||||
try:
|
||||
global callback_list, add_breadcrumb, user_logger_fn, Logging
|
||||
|
|
@ -2457,10 +2523,12 @@ def function_setup(
|
|||
len(litellm.input_callback) > 0
|
||||
or len(litellm.success_callback) > 0
|
||||
or len(litellm.failure_callback) > 0
|
||||
) and len(callback_list) == 0:
|
||||
) and len(
|
||||
callback_list # type: ignore
|
||||
) == 0: # type: ignore
|
||||
callback_list = list(
|
||||
set(
|
||||
litellm.input_callback
|
||||
litellm.input_callback # type: ignore
|
||||
+ litellm.success_callback
|
||||
+ litellm.failure_callback
|
||||
)
|
||||
|
|
@ -2469,7 +2537,7 @@ def function_setup(
|
|||
## ASYNC CALLBACKS
|
||||
if len(litellm.input_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.input_callback):
|
||||
for index, callback in enumerate(litellm.input_callback): # type: ignore
|
||||
if inspect.iscoroutinefunction(callback):
|
||||
litellm._async_input_callback.append(callback)
|
||||
removed_async_items.append(index)
|
||||
|
|
@ -2480,7 +2548,7 @@ def function_setup(
|
|||
|
||||
if len(litellm.success_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.success_callback):
|
||||
for index, callback in enumerate(litellm.success_callback): # type: ignore
|
||||
if inspect.iscoroutinefunction(callback):
|
||||
litellm._async_success_callback.append(callback)
|
||||
removed_async_items.append(index)
|
||||
|
|
@ -2496,7 +2564,7 @@ def function_setup(
|
|||
|
||||
if len(litellm.failure_callback) > 0:
|
||||
removed_async_items = []
|
||||
for index, callback in enumerate(litellm.failure_callback):
|
||||
for index, callback in enumerate(litellm.failure_callback): # type: ignore
|
||||
if inspect.iscoroutinefunction(callback):
|
||||
litellm._async_failure_callback.append(callback)
|
||||
removed_async_items.append(index)
|
||||
|
|
@ -2539,7 +2607,7 @@ def function_setup(
|
|||
user_logger_fn = kwargs["logger_fn"]
|
||||
# INIT LOGGER - for user-specified integrations
|
||||
model = args[0] if len(args) > 0 else kwargs.get("model", None)
|
||||
call_type = original_function.__name__
|
||||
call_type = original_function
|
||||
if (
|
||||
call_type == CallTypes.completion.value
|
||||
or call_type == CallTypes.acompletion.value
|
||||
|
|
@ -2721,7 +2789,7 @@ def client(original_function):
|
|||
try:
|
||||
if logging_obj is None:
|
||||
logging_obj, kwargs = function_setup(
|
||||
original_function, rules_obj, start_time, *args, **kwargs
|
||||
original_function.__name__, rules_obj, start_time, *args, **kwargs
|
||||
)
|
||||
kwargs["litellm_logging_obj"] = logging_obj
|
||||
|
||||
|
|
@ -3030,7 +3098,7 @@ def client(original_function):
|
|||
try:
|
||||
if logging_obj is None:
|
||||
logging_obj, kwargs = function_setup(
|
||||
original_function, rules_obj, start_time, *args, **kwargs
|
||||
original_function.__name__, rules_obj, start_time, *args, **kwargs
|
||||
)
|
||||
kwargs["litellm_logging_obj"] = logging_obj
|
||||
|
||||
|
|
@ -5265,7 +5333,8 @@ def get_optional_params(
|
|||
optional_params["tools"] = tools
|
||||
if tool_choice is not None:
|
||||
optional_params["tool_choice"] = tool_choice
|
||||
|
||||
if response_format is not None:
|
||||
optional_params["response_format"] = response_format
|
||||
# check safe_mode, random_seed: https://docs.mistral.ai/api/#operation/createChatCompletion
|
||||
safe_mode = passed_params.pop("safe_mode", None)
|
||||
random_seed = passed_params.pop("random_seed", None)
|
||||
|
|
@ -5277,6 +5346,7 @@ def get_optional_params(
|
|||
optional_params["extra_body"] = (
|
||||
extra_body # openai client supports `extra_body` param
|
||||
)
|
||||
|
||||
elif custom_llm_provider == "groq":
|
||||
supported_params = get_supported_openai_params(
|
||||
model=model, custom_llm_provider=custom_llm_provider
|
||||
|
|
@ -7033,9 +7103,10 @@ def convert_to_model_response_object(
|
|||
model_response_object.model = response_object["model"]
|
||||
|
||||
if start_time is not None and end_time is not None:
|
||||
model_response_object._response_ms = ( # type: ignore
|
||||
end_time - start_time
|
||||
).total_seconds() * 1000
|
||||
if isinstance(start_time, type(end_time)):
|
||||
model_response_object._response_ms = ( # type: ignore
|
||||
end_time - start_time
|
||||
).total_seconds() * 1000
|
||||
|
||||
if hidden_params is not None:
|
||||
model_response_object._hidden_params = hidden_params
|
||||
|
|
@ -10120,12 +10191,23 @@ class CustomStreamWrapper:
|
|||
model_response.id = original_chunk.id
|
||||
self.response_id = original_chunk.id
|
||||
if len(original_chunk.choices) > 0:
|
||||
try:
|
||||
delta = dict(original_chunk.choices[0].delta)
|
||||
print_verbose(f"original delta: {delta}")
|
||||
model_response.choices[0].delta = Delta(**delta)
|
||||
except Exception as e:
|
||||
model_response.choices[0].delta = Delta()
|
||||
choices = []
|
||||
for idx, choice in enumerate(original_chunk.choices):
|
||||
try:
|
||||
if isinstance(choice, BaseModel):
|
||||
try:
|
||||
choice_json = choice.model_dump()
|
||||
except Exception as e:
|
||||
choice_json = choice.dict()
|
||||
choice_json.pop(
|
||||
"finish_reason", None
|
||||
) # for mistral etc. which return a value in their last chunk (not-openai compatible).
|
||||
print_verbose(f"choice_json: {choice_json}")
|
||||
choices.append(StreamingChoices(**choice_json))
|
||||
except Exception as e:
|
||||
choices.append(StreamingChoices())
|
||||
print_verbose(f"choices in streaming: {choices}")
|
||||
model_response.choices = choices
|
||||
else:
|
||||
return
|
||||
model_response.system_fingerprint = (
|
||||
|
|
@ -10170,11 +10252,11 @@ class CustomStreamWrapper:
|
|||
)
|
||||
self.holding_chunk = ""
|
||||
# if delta is None
|
||||
is_delta_empty = self.is_delta_empty(
|
||||
_is_delta_empty = self.is_delta_empty(
|
||||
delta=model_response.choices[0].delta
|
||||
)
|
||||
|
||||
if is_delta_empty:
|
||||
if _is_delta_empty:
|
||||
# get any function call arguments
|
||||
model_response.choices[0].finish_reason = map_finish_reason(
|
||||
finish_reason=self.received_finish_reason
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[tool.poetry]
|
||||
name = "litellm"
|
||||
version = "1.35.27"
|
||||
version = "1.35.29"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
authors = ["BerriAI"]
|
||||
license = "MIT"
|
||||
|
|
@ -80,7 +80,7 @@ requires = ["poetry-core", "wheel"]
|
|||
build-backend = "poetry.core.masonry.api"
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.35.27"
|
||||
version = "1.35.29"
|
||||
version_files = [
|
||||
"pyproject.toml:^version"
|
||||
]
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ fastapi==0.100.0 # server dep
|
|||
backoff==2.2.1 # server dep
|
||||
pyyaml==6.0.0 # server dep
|
||||
uvicorn==0.29.0 # server dep
|
||||
gunicorn==21.2.0 # server dep
|
||||
gunicorn==22.0.0 # server dep
|
||||
boto3==1.34.34 # aws bedrock/sagemaker calls
|
||||
redis==5.0.0 # caching
|
||||
numpy==1.24.3 # semantic caching
|
||||
|
|
|
|||
|
|
@ -523,7 +523,9 @@ async def test_key_info_spend_values_streaming():
|
|||
)
|
||||
rounded_response_cost = round(response_cost, 8)
|
||||
rounded_key_info_spend = round(key_info["info"]["spend"], 8)
|
||||
assert rounded_response_cost == rounded_key_info_spend
|
||||
assert (
|
||||
rounded_response_cost == rounded_key_info_spend
|
||||
), f"Expected={rounded_response_cost}, Got={rounded_key_info_spend}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/60d9f441227ccc7e.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
|
||||
!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e](n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){return"static/css/5e699db73bf6f8c2.css"},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/ui/_next/",i={272:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(272!=e){var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}else i[e]=0}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1 +1 @@
|
|||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-50c1dadc6557c101.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-dafd44dfa2da140c.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e49705773ae41779.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-50c1dadc6557c101.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/60d9f441227ccc7e.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[82332,[\"127\",\"static/chunks/127-efd0436630e294eb.js\",\"931\",\"static/chunks/app/page-525d83925fd5350b.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/60d9f441227ccc7e.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"Csz8BqWx6JEoKsgLqCeCt\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-ccae12a25017afa5.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-dafd44dfa2da140c.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-e49705773ae41779.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-ccae12a25017afa5.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/5e699db73bf6f8c2.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[27125,[\"447\",\"static/chunks/447-9f8d32190ff7d16d.js\",\"931\",\"static/chunks/app/page-781ca5f151d78d1d.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/5e699db73bf6f8c2.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"PtTtxXIYvdjQsvRgdITlk\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
2:I[77831,[],""]
|
||||
3:I[82332,["127","static/chunks/127-efd0436630e294eb.js","931","static/chunks/app/page-525d83925fd5350b.js"],""]
|
||||
3:I[27125,["447","static/chunks/447-9f8d32190ff7d16d.js","931","static/chunks/app/page-781ca5f151d78d1d.js"],""]
|
||||
4:I[5613,[],""]
|
||||
5:I[31778,[],""]
|
||||
0:["Csz8BqWx6JEoKsgLqCeCt",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/60d9f441227ccc7e.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
0:["PtTtxXIYvdjQsvRgdITlk",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/5e699db73bf6f8c2.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
|
||||
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
|
||||
1:null
|
||||
|
|
|
|||
|
|
@ -261,6 +261,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
<Form.Item
|
||||
label="tpm"
|
||||
name="tpm"
|
||||
tooltip="int (optional) - Tokens limit for this deployment: in tokens per minute (tpm). Find this information on your model/providers website"
|
||||
>
|
||||
<InputNumber min={0} step={1} />
|
||||
|
||||
|
|
@ -269,6 +270,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
<Form.Item
|
||||
label="rpm"
|
||||
name="rpm"
|
||||
tooltip="int (optional) - Rate limit for this deployment: in requests per minute (rpm). Find this information on your model/providers website"
|
||||
>
|
||||
<InputNumber min={0} step={1} />
|
||||
</Form.Item>
|
||||
|
|
@ -283,6 +285,24 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
|
|||
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label="timeout"
|
||||
name="timeout"
|
||||
tooltip="int (optional) - Timeout in seconds for LLM requests (Defaults to 600 seconds)"
|
||||
>
|
||||
<InputNumber min={0} step={1} />
|
||||
|
||||
</Form.Item>
|
||||
|
||||
<Form.Item
|
||||
label="stream_timeout"
|
||||
name="stream_timeout"
|
||||
tooltip="int (optional) - Timeout for stream requests (seconds)"
|
||||
>
|
||||
<InputNumber min={0} step={1} />
|
||||
|
||||
</Form.Item>
|
||||
|
||||
|
||||
|
||||
<Form.Item
|
||||
|
|
|
|||
|
|
@ -329,6 +329,42 @@ export const userInfoCall = async (
|
|||
}
|
||||
};
|
||||
|
||||
|
||||
export const teamInfoCall = async (
|
||||
accessToken: String,
|
||||
teamID: String | null,
|
||||
) => {
|
||||
try {
|
||||
let url = proxyBaseUrl ? `${proxyBaseUrl}/team/info` : `/team/info`;
|
||||
if (teamID) {
|
||||
url = `${url}?team_id=${teamID}`;
|
||||
}
|
||||
console.log("in teamInfoCall");
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
Authorization: `Bearer ${accessToken}`,
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errorData = await response.text();
|
||||
message.error(errorData, 20);
|
||||
throw new Error("Network response was not ok");
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
console.log("API Response:", data);
|
||||
return data;
|
||||
// Handle success - you might want to update some state or UI based on the created key
|
||||
} catch (error) {
|
||||
console.error("Failed to create key:", error);
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
export const getTotalSpendCall = async (
|
||||
accessToken: String,
|
||||
) => {
|
||||
|
|
@ -1238,7 +1274,7 @@ export const serviceHealthCheck= async (accessToken: String, service: String) =>
|
|||
}
|
||||
|
||||
const data = await response.json();
|
||||
message.success(`Test request to ${service} made - check logs on ${service} dashboard!`);
|
||||
message.success(`Test request to ${service} made - check logs/alerts on ${service} to verify`);
|
||||
// You can add additional logic here based on the response if needed
|
||||
return data;
|
||||
} catch (error) {
|
||||
|
|
|
|||
|
|
@ -15,7 +15,13 @@ import {
|
|||
Grid,
|
||||
Button,
|
||||
TextInput,
|
||||
Switch,
|
||||
Col,
|
||||
TabPanel,
|
||||
TabPanels,
|
||||
TabGroup,
|
||||
TabList,
|
||||
Tab
|
||||
} from "@tremor/react";
|
||||
import { getCallbacksCall, setCallbacksCall, serviceHealthCheck } from "./networking";
|
||||
import { Modal, Form, Input, Select, Button as Button2, message } from "antd";
|
||||
|
|
@ -45,10 +51,29 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
userID,
|
||||
}) => {
|
||||
const [callbacks, setCallbacks] = useState<any[]>([]);
|
||||
const [alerts, setAlerts] = useState<any[]>([]);
|
||||
const [isModalVisible, setIsModalVisible] = useState(false);
|
||||
const [form] = Form.useForm();
|
||||
const [selectedCallback, setSelectedCallback] = useState<string | null>(null);
|
||||
const [selectedAlertValues, setSelectedAlertValues] = useState([]);
|
||||
const [catchAllWebhookURL, setCatchAllWebhookURL] = useState<string>("");
|
||||
const [alertToWebhooks, setAlertToWebhooks] = useState<Record<string, string>>({});
|
||||
const [activeAlerts, setActiveAlerts] = useState<string[]>([]);
|
||||
|
||||
|
||||
const handleSwitchChange = (alertName: string) => {
|
||||
if (activeAlerts.includes(alertName)) {
|
||||
setActiveAlerts(activeAlerts.filter((alert) => alert !== alertName));
|
||||
} else {
|
||||
setActiveAlerts([...activeAlerts, alertName]);
|
||||
}
|
||||
};
|
||||
const alerts_to_UI_NAME: Record<string, string> = {
|
||||
"llm_exceptions": "LLM Exceptions",
|
||||
"llm_too_slow": "LLM Responses Too Slow",
|
||||
"llm_requests_hanging": "LLM Requests Hanging",
|
||||
"budget_alerts": "Budget Alerts (API Keys, Users)"
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (!accessToken || !userRole || !userID) {
|
||||
|
|
@ -56,11 +81,35 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
}
|
||||
getCallbacksCall(accessToken, userID, userRole).then((data) => {
|
||||
console.log("callbacks", data);
|
||||
let callbacks_data = data.data;
|
||||
let callbacks_data = data.callbacks;
|
||||
setCallbacks(callbacks_data);
|
||||
|
||||
let alerts_data = data.alerts;
|
||||
console.log("alerts_data", alerts_data);
|
||||
if (alerts_data) {
|
||||
if (alerts_data.length > 0) {
|
||||
let _alert_info = alerts_data[0];
|
||||
console.log("_alert_info", _alert_info);
|
||||
let catch_all_webhook = _alert_info.variables.SLACK_WEBHOOK_URL;
|
||||
console.log("catch_all_webhook", catch_all_webhook);
|
||||
|
||||
let active_alerts = _alert_info.active_alerts;
|
||||
setActiveAlerts(active_alerts);
|
||||
setCatchAllWebhookURL(catch_all_webhook);
|
||||
setAlertToWebhooks(_alert_info.alerts_to_webhook);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
setAlerts(alerts_data);
|
||||
});
|
||||
}, [accessToken, userRole, userID]);
|
||||
|
||||
|
||||
const isAlertOn = (alertName: string) => {
|
||||
return activeAlerts && activeAlerts.includes(alertName);
|
||||
}
|
||||
|
||||
const handleAddCallback = () => {
|
||||
console.log("Add callback clicked");
|
||||
setIsModalVisible(true);
|
||||
|
|
@ -78,6 +127,40 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
console.log('Selected values:', values);
|
||||
};
|
||||
|
||||
const handleSaveAlerts = () => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
}
|
||||
|
||||
const updatedAlertToWebhooks: Record<string, string> = {};
|
||||
Object.entries(alerts_to_UI_NAME).forEach(([key, value]) => {
|
||||
const webhookInput = document.querySelector(`input[name="${key}"]`) as HTMLInputElement;
|
||||
console.log("key", key);
|
||||
console.log("webhookInput", webhookInput);
|
||||
const newWebhookValue = webhookInput?.value || '';
|
||||
console.log("newWebhookValue", newWebhookValue);
|
||||
updatedAlertToWebhooks[key] = newWebhookValue;
|
||||
});
|
||||
|
||||
console.log("updatedAlertToWebhooks", updatedAlertToWebhooks);
|
||||
|
||||
const payload = {
|
||||
general_settings: {
|
||||
alert_to_webhook_url: updatedAlertToWebhooks,
|
||||
alert_types: activeAlerts
|
||||
},
|
||||
};
|
||||
|
||||
console.log("payload", payload);
|
||||
|
||||
try {
|
||||
setCallbacksCall(accessToken, payload);
|
||||
} catch (error) {
|
||||
message.error('Failed to update alerts: ' + error, 20);
|
||||
}
|
||||
|
||||
message.success('Alerts updated successfully');
|
||||
};
|
||||
const handleSaveChanges = (callback: any) => {
|
||||
if (!accessToken) {
|
||||
return;
|
||||
|
|
@ -92,9 +175,6 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
|
||||
const payload = {
|
||||
environment_variables: updatedVariables,
|
||||
general_settings: {
|
||||
alert_types: selectedAlertValues
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
|
|
@ -186,71 +266,114 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
return (
|
||||
<div className="w-full mx-4">
|
||||
<Grid numItems={1} className="gap-2 p-8 w-full mt-2">
|
||||
<Title>Logging Callbacks</Title>
|
||||
<Card >
|
||||
<Table>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeaderCell>Callback</TableHeaderCell>
|
||||
<TableHeaderCell>Callback Env Vars</TableHeaderCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{callbacks.map((callback, index) => (
|
||||
<TableRow key={index}>
|
||||
<TableCell>
|
||||
<Badge color="emerald">{callback.name}</Badge>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<ul>
|
||||
{Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => (
|
||||
<li key={key}>
|
||||
<Text className="mt-2">{key}</Text>
|
||||
{key === "LANGFUSE_HOST" ? (
|
||||
<p>default value=https://cloud.langfuse.com</p>
|
||||
) : (
|
||||
<div></div>
|
||||
)}
|
||||
<TextInput name={key} defaultValue={value as string} type="password" />
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
{callback.all_alert_types && (
|
||||
<div>
|
||||
<Text className="mt-2">Alerting Types</Text>
|
||||
<Select
|
||||
mode="multiple"
|
||||
style={{ width: '100%' }}
|
||||
placeholder="Select Alerting Types"
|
||||
optionLabelProp="label"
|
||||
onChange={handleChange}
|
||||
defaultValue={callback.alerting_types}
|
||||
>
|
||||
{callback.all_alert_types.map((type: string) => (
|
||||
<Select.Option key={type} value={type} label={type}>
|
||||
{type}
|
||||
</Select.Option>
|
||||
))}
|
||||
</Select>
|
||||
</div>
|
||||
)}
|
||||
<Button className="mt-2" onClick={() => handleSaveChanges(callback)}>
|
||||
<TabGroup>
|
||||
<TabList variant="line" defaultValue="1">
|
||||
<Tab value="1">Logging Callbacks</Tab>
|
||||
<Tab value="2">Alerting</Tab>
|
||||
</TabList>
|
||||
<TabPanels>
|
||||
<TabPanel>
|
||||
<Card >
|
||||
<Table>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeaderCell>Callback</TableHeaderCell>
|
||||
<TableHeaderCell>Callback Env Vars</TableHeaderCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
<TableBody>
|
||||
{callbacks.map((callback, index) => (
|
||||
<TableRow key={index}>
|
||||
<TableCell>
|
||||
<Badge color="emerald">{callback.name}</Badge>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<ul>
|
||||
{Object.entries(callback.variables ?? {}).filter(([key, value]) => value !== null).map(([key, value]) => (
|
||||
<li key={key}>
|
||||
<Text className="mt-2">{key}</Text>
|
||||
{key === "LANGFUSE_HOST" ? (
|
||||
<p>default value=https://cloud.langfuse.com</p>
|
||||
) : (
|
||||
<div></div>
|
||||
)}
|
||||
<TextInput name={key} defaultValue={value as string} type="password" />
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<Button className="mt-2" onClick={() => handleSaveChanges(callback)}>
|
||||
Save Changes
|
||||
</Button>
|
||||
<Button onClick={() => serviceHealthCheck(accessToken, callback.name)} className="mx-2">
|
||||
Test Callback
|
||||
</Button>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
<Button size="xs" className="mt-2" onClick={handleAddCallback}>
|
||||
Add Callback
|
||||
</Button>
|
||||
|
||||
</Card>
|
||||
</TabPanel>
|
||||
|
||||
<TabPanel>
|
||||
|
||||
<Card>
|
||||
<Text className="my-2">Alerts are only supported for Slack Webhook URLs. Get your webhook urls from <a href="https://api.slack.com/messaging/webhooks" target="_blank" style={{color: 'blue'}}>here</a></Text>
|
||||
<Table>
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeaderCell></TableHeaderCell>
|
||||
<TableHeaderCell></TableHeaderCell>
|
||||
<TableHeaderCell>Slack Webhook URL</TableHeaderCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
|
||||
<TableBody>
|
||||
{Object.entries(alerts_to_UI_NAME).map(([key, value], index) => (
|
||||
<TableRow key={index}>
|
||||
<TableCell>
|
||||
<Switch
|
||||
id="switch"
|
||||
name="switch"
|
||||
checked={isAlertOn(key)}
|
||||
onChange={() => handleSwitchChange(key)}
|
||||
/>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Text>{value}</Text>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<TextInput name={key} type="password" defaultValue={alertToWebhooks && alertToWebhooks[key] ? alertToWebhooks[key] : catchAllWebhookURL as string}>
|
||||
|
||||
</TextInput>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
<Button size="xs" className="mt-2" onClick={handleSaveAlerts}>
|
||||
Save Changes
|
||||
</Button>
|
||||
<Button onClick={() => serviceHealthCheck(accessToken, callback.name)} className="mx-2">
|
||||
Test Callback
|
||||
</Button>
|
||||
</TableCell>
|
||||
</TableRow>
|
||||
))}
|
||||
</TableBody>
|
||||
</Table>
|
||||
<Button size="xs" className="mt-2" onClick={handleAddCallback}>
|
||||
Add Callback
|
||||
</Button>
|
||||
|
||||
|
||||
<Button onClick={() => serviceHealthCheck(accessToken, "slack")} className="mx-2">
|
||||
Test Alerts
|
||||
</Button>
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
</Card>
|
||||
|
||||
|
||||
</TabPanel>
|
||||
</TabPanels>
|
||||
</TabGroup>
|
||||
|
||||
|
||||
</Grid>
|
||||
|
||||
<Modal
|
||||
|
|
@ -269,7 +392,6 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
>
|
||||
<Select onChange={handleCallbackChange}>
|
||||
<Select.Option value="langfuse">langfuse</Select.Option>
|
||||
<Select.Option value="slack">slack alerting</Select.Option>
|
||||
</Select>
|
||||
</Form.Item>
|
||||
|
||||
|
|
@ -297,18 +419,6 @@ const Settings: React.FC<SettingsPageProps> = ({
|
|||
</>
|
||||
)}
|
||||
|
||||
{selectedCallback === 'slack' && (
|
||||
<Form.Item
|
||||
label="SLACK_WEBHOOK_URL"
|
||||
name="slackWebhookUrl"
|
||||
rules={[
|
||||
{ required: true, message: "Please enter the Slack webhook URL" },
|
||||
]}
|
||||
>
|
||||
<TextInput/>
|
||||
</Form.Item>
|
||||
)}
|
||||
|
||||
<div style={{ textAlign: "right", marginTop: "10px" }}>
|
||||
<Button2 htmlType="submit">Save</Button2>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import React, { useState, useEffect } from "react";
|
||||
import Link from "next/link";
|
||||
import { Typography } from "antd";
|
||||
import { teamDeleteCall, teamUpdateCall } from "./networking";
|
||||
import { teamDeleteCall, teamUpdateCall, teamInfoCall } from "./networking";
|
||||
import { InformationCircleIcon, PencilAltIcon, PencilIcon, StatusOnlineIcon, TrashIcon } from "@heroicons/react/outline";
|
||||
import {
|
||||
Button as Button2,
|
||||
|
|
@ -73,6 +73,9 @@ const Team: React.FC<TeamProps> = ({
|
|||
const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false);
|
||||
const [teamToDelete, setTeamToDelete] = useState<string | null>(null);
|
||||
|
||||
// store team info as {"team_id": team_info_object}
|
||||
const [perTeamInfo, setPerTeamInfo] = useState<Record<string, any>>({});
|
||||
|
||||
|
||||
const EditTeamModal: React.FC<EditTeamModalProps> = ({ visible, onCancel, team, onSubmit }) => {
|
||||
const [form] = Form.useForm();
|
||||
|
|
@ -271,9 +274,39 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
|
|||
console.error("Error fetching user models:", error);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
const fetchTeamInfo = async () => {
|
||||
try {
|
||||
if (userID === null || userRole === null || accessToken === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (teams === null) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log("fetching team info:");
|
||||
|
||||
|
||||
let _team_id_to_info: Record<string, any> = {};
|
||||
for (let i = 0; i < teams?.length; i++) {
|
||||
let _team_id = teams[i].team_id;
|
||||
const teamInfo = await teamInfoCall(accessToken, _team_id);
|
||||
console.log("teamInfo response:", teamInfo);
|
||||
if (teamInfo !== null) {
|
||||
_team_id_to_info = {..._team_id_to_info, [_team_id]: teamInfo};
|
||||
}
|
||||
}
|
||||
setPerTeamInfo(_team_id_to_info);
|
||||
} catch (error) {
|
||||
console.error("Error fetching team info:", error);
|
||||
}
|
||||
};
|
||||
|
||||
fetchUserModels();
|
||||
}, [accessToken, userID, userRole]);
|
||||
fetchTeamInfo();
|
||||
}, [accessToken, userID, userRole, teams]);
|
||||
|
||||
const handleCreate = async (formValues: Record<string, any>) => {
|
||||
try {
|
||||
|
|
@ -346,6 +379,7 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
|
|||
<TableHeaderCell>Budget (USD)</TableHeaderCell>
|
||||
<TableHeaderCell>Models</TableHeaderCell>
|
||||
<TableHeaderCell>TPM / RPM Limits</TableHeaderCell>
|
||||
<TableHeaderCell>Info</TableHeaderCell>
|
||||
</TableRow>
|
||||
</TableHead>
|
||||
|
||||
|
|
@ -381,6 +415,7 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
|
|||
</div>
|
||||
) : null}
|
||||
</TableCell>
|
||||
|
||||
|
||||
<TableCell style={{ maxWidth: "4px", whiteSpace: "pre-wrap", overflow: "hidden" }}>
|
||||
<Text>
|
||||
|
|
@ -390,6 +425,10 @@ const handleEditSubmit = async (formValues: Record<string, any>) => {
|
|||
{team.rpm_limit ? team.rpm_limit : "Unlimited"}
|
||||
</Text>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Text>{perTeamInfo && team.team_id && perTeamInfo[team.team_id] && perTeamInfo[team.team_id].keys && perTeamInfo[team.team_id].keys.length} Keys</Text>
|
||||
<Text>{perTeamInfo && team.team_id && perTeamInfo[team.team_id] && perTeamInfo[team.team_id].team_info && perTeamInfo[team.team_id].team_info.members_with_roles && perTeamInfo[team.team_id].team_info.members_with_roles.length} Members</Text>
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
<Icon
|
||||
icon={PencilAltIcon}
|
||||
|
|
|
|||
|
|
@ -183,11 +183,12 @@ const ViewKeyTable: React.FC<ViewKeyTableProps> = ({
|
|||
const errorModels = value.filter((model: string) => (
|
||||
!keyTeam.models.includes(model) &&
|
||||
model !== "all-team-models" &&
|
||||
model !== "all-proxy-models"
|
||||
model !== "all-proxy-models" &&
|
||||
!keyTeam.models.includes("all-proxy-models")
|
||||
));
|
||||
console.log(`errorModels: ${errorModels}`)
|
||||
if (errorModels.length > 0) {
|
||||
return Promise.reject(`Some models are not part of the new team\'s models - ${errorModels}`);
|
||||
return Promise.reject(`Some models are not part of the new team\'s models - ${errorModels}Team models: ${keyTeam.models}`);
|
||||
} else {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
|
@ -425,7 +426,7 @@ const ViewKeyTable: React.FC<ViewKeyTableProps> = ({
|
|||
return (
|
||||
<div>
|
||||
<Card className="w-full mx-auto flex-auto overflow-y-auto max-h-[50vh] mb-4 mt-2">
|
||||
<Table className="mt-5">
|
||||
<Table className="mt-5 max-h-[300px] min-h-[300px]">
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeaderCell>Key Alias</TableHeaderCell>
|
||||
|
|
|
|||
|
|
@ -153,9 +153,13 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
|
|||
|
||||
return (
|
||||
<div style={{ width: "100%" }}>
|
||||
<Grid className="gap-2 p-2 h-[75vh] w-full mt-8">
|
||||
<Grid className="gap-2 p-2 h-[80vh] w-full mt-8">
|
||||
<CreateUser userID={userID} accessToken={accessToken} teams={teams}/>
|
||||
<Card className="w-full mx-auto flex-auto overflow-y-auto max-h-[50vh] mb-4">
|
||||
<Card className="w-full mx-auto flex-auto overflow-y-auto max-h-[80vh] mb-4">
|
||||
<div className="mb-4 mt-1">
|
||||
<Text><b>Key Owners: </b> Users on LiteLLM that created API Keys. Automatically tracked by LiteLLM</Text>
|
||||
<Text className="mt-1"><b>End Users: </b>End Users of your LLM API calls. Tracked When a `user` param is passed in your LLM calls</Text>
|
||||
</div>
|
||||
<TabGroup>
|
||||
<TabList variant="line" defaultValue="1">
|
||||
<Tab value="1">Key Owners</Tab>
|
||||
|
|
@ -163,6 +167,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
|
|||
</TabList>
|
||||
<TabPanels>
|
||||
<TabPanel>
|
||||
|
||||
<Table className="mt-5">
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
|
|
@ -220,8 +225,8 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
|
|||
{keys?.map((key: any, index: number) => {
|
||||
if (
|
||||
key &&
|
||||
key["key_name"] !== null &&
|
||||
key["key_name"].length > 0
|
||||
key["key_alias"] !== null &&
|
||||
key["key_alias"].length > 0
|
||||
) {
|
||||
return (
|
||||
<SelectItem
|
||||
|
|
@ -229,7 +234,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
|
|||
value={String(index)}
|
||||
onClick={() => onKeyClick(key["token"])}
|
||||
>
|
||||
{key["key_name"]}
|
||||
{key["key_alias"]}
|
||||
</SelectItem>
|
||||
);
|
||||
}
|
||||
|
|
@ -237,7 +242,7 @@ const ViewUserDashboard: React.FC<ViewUserDashboardProps> = ({
|
|||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
<Table>
|
||||
<Table className="max-h-[70vh] min-h-[500px]">
|
||||
<TableHead>
|
||||
<TableRow>
|
||||
<TableHeaderCell>End User</TableHeaderCell>
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue