Merge branch 'BerriAI:main' into ollama-image-handling

docs/my-website/docs/providers/anthropic.md

@@ -9,6 +9,12 @@ LiteLLM supports
 - `claude-2.1`
 - `claude-instant-1.2`
 
+:::info
+
+Anthropic API fails requests when `max_tokens` are not passed. Due to this litellm passes `max_tokens=4096` when no `max_tokens` are passed
+
+:::
+
 ## API Keys
 
 ```python

docs/my-website/docs/proxy/email.md

@@ -2,12 +2,6 @@ import Image from '@theme/IdealImage';
 
 # ✨ 📧 Email Notifications 
 
-:::info
-
-This is an Enterprise only feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
-
-:::
-
 Send an Email to your users when:
 - A Proxy API Key is created for them 
 - Their API Key crosses it's Budget 
@@ -38,6 +32,12 @@ That's it ! start your proxy
 
 ## Customizing Email Branding
 
+:::info
+
+Customizing Email Branding is an Enterprise Feature [Get in touch with us for a Free Trial](https://calendly.com/d/4mp-gd3-k5k/litellm-1-1-onboarding-chat)
+
+:::
+
 LiteLLM allows you to customize the:
 - Logo on the Email
 - Email support contact 

docs/my-website/docs/proxy/enterprise.md

@@ -21,6 +21,7 @@ Features:
 - ✅ Don't log/store specific requests to Langfuse, Sentry, etc. (eg confidential LLM requests)
 - ✅ Tracking Spend for Custom Tags
 - ✅ Custom Branding + Routes on Swagger Docs
+- ✅ Audit Logs for `Created At, Created By` when Models Added
 
 
 ## Content Moderation

docs/my-website/docs/proxy/users.md

@@ -13,7 +13,7 @@ Requirements:
 You can set budgets at 3 levels: 
 - For the proxy 
 - For an internal user 
-- For an end-user
+- For a customer (end-user)
 - For a key
 - For a key (model specific budgets)
 
@@ -173,7 +173,7 @@ curl --location 'http://localhost:4000/chat/completions' \
 ```
 
 </TabItem>
-<TabItem value="per-user-chat" label="For End User">
+<TabItem value="per-user-chat" label="For Customers">
 
 Use this to budget `user` passed to `/chat/completions`, **without needing to create a key for every user**
 
@@ -452,7 +452,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \
 ```
 
 </TabItem>
-<TabItem value="per-end-user" label="For End User">
+<TabItem value="per-end-user" label="For customers">
 
 :::info 
 
@@ -477,12 +477,12 @@ curl --location 'http://0.0.0.0:4000/budget/new' \
 ```
 
 
-#### Step 2. Create `End-User` with Budget
+#### Step 2. Create `Customer` with Budget
 
-We use `budget_id="free-tier"` from Step 1 when creating this new end user
+We use `budget_id="free-tier"` from Step 1 when creating this new customers
 
 ```shell
-curl --location 'http://0.0.0.0:4000/end_user/new' \
+curl --location 'http://0.0.0.0:4000/customer/new' \
 --header 'Authorization: Bearer sk-1234' \
 --header 'Content-Type: application/json' \
 --data '{
@@ -492,7 +492,7 @@ curl --location 'http://0.0.0.0:4000/end_user/new' \
 ```
 
 
-#### Step 3. Pass end user id in `/chat/completions` requests
+#### Step 3. Pass `user_id` id in `/chat/completions` requests
 
 Pass the `user_id` from Step 2 as `user="palantir"` 
 

docs/my-website/docs/troubleshoot.md

@@ -9,12 +9,3 @@ Our emails ✉️ ishaan@berri.ai / krrish@berri.ai
 
 [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) 
 
-## Stable Version
-
-If you're running into problems with installation / Usage 
-Use the stable version of litellm 
-
-```shell
-pip install litellm==0.1.819
-```
-

litellm/__init__.py

@@ -766,7 +766,12 @@ from .llms.bedrock import (
     AmazonMistralConfig,
     AmazonBedrockGlobalConfig,
 )
-from .llms.openai import OpenAIConfig, OpenAITextCompletionConfig, MistralConfig
+from .llms.openai import (
+    OpenAIConfig,
+    OpenAITextCompletionConfig,
+    MistralConfig,
+    DeepInfraConfig,
+)
 from .llms.azure import AzureOpenAIConfig, AzureOpenAIError
 from .llms.watsonx import IBMWatsonXAIConfig
 from .main import *  # type: ignore

litellm/integrations/slack_alerting.py

@@ -41,10 +41,6 @@ class ProviderRegionOutageModel(BaseOutageModel):
 
 # we use this for the email header, please send a test email if you change this. verify it looks good on email
 LITELLM_LOGO_URL = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
-EMAIL_LOGO_URL = os.getenv(
-    "SMTP_SENDER_LOGO", "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
-)
-EMAIL_SUPPORT_CONTACT = os.getenv("EMAIL_SUPPORT_CONTACT", "support@berri.ai")
 
 
 class LiteLLMBase(BaseModel):
@@ -1147,21 +1143,34 @@ Model Info:
 
         return False
 
+    async def _check_if_using_premium_email_feature(
+        self,
+        premium_user: bool,
+        email_logo_url: Optional[str] = None,
+        email_support_contact: Optional[str] = None,
+    ):
+        from litellm.proxy.proxy_server import premium_user
+        from litellm.proxy.proxy_server import CommonProxyErrors
+
+        if premium_user is not True:
+            if email_logo_url is not None or email_support_contact is not None:
+                raise ValueError(
+                    f"Trying to Customize Email Alerting\n {CommonProxyErrors.not_premium_user.value}"
+                )
+
     async def send_key_created_email(self, webhook_event: WebhookEvent) -> bool:
         from litellm.proxy.utils import send_email
 
         if self.alerting is None or "email" not in self.alerting:
             # do nothing if user does not want email alerts
             return False
+        from litellm.proxy.proxy_server import premium_user, prisma_client
 
-        # make sure this is a premium user
-        from litellm.proxy.proxy_server import premium_user
-        from litellm.proxy.proxy_server import CommonProxyErrors, prisma_client
-
-        if premium_user != True:
-            raise Exception(
-                f"Trying to use Email Alerting on key creation\n {CommonProxyErrors.not_premium_user.value}"
-            )
+        email_logo_url = os.getenv("SMTP_SENDER_LOGO", None)
+        email_support_contact = os.getenv("EMAIL_SUPPORT_CONTACT", None)
+        await self._check_if_using_premium_email_feature(
+            premium_user, email_logo_url, email_support_contact
+        )
 
         event_name = webhook_event.event_message
         recipient_email = webhook_event.user_email
@@ -1188,7 +1197,7 @@ Model Info:
                 "Trying to send email alert to no recipient", extra=webhook_event.dict()
             )
         email_html_content = f"""
-            <img src="{EMAIL_LOGO_URL}" alt="LiteLLM Logo" width="150" height="50" />
+            <img src="{email_logo_url}" alt="LiteLLM Logo" width="150" height="50" />
 
             <p> Hi {recipient_email}, <br/>
  
@@ -1223,7 +1232,7 @@ Model Info:
             </pre>
 
 
-            If you have any questions, please send an email to {EMAIL_SUPPORT_CONTACT} <br /> <br />
+            If you have any questions, please send an email to {email_support_contact} <br /> <br />
 
             Best, <br />
             The LiteLLM team <br />
@@ -1254,6 +1263,14 @@ Model Info:
         """
         from litellm.proxy.utils import send_email
 
+        from litellm.proxy.proxy_server import premium_user, prisma_client
+
+        email_logo_url = os.getenv("SMTP_SENDER_LOGO", None)
+        email_support_contact = os.getenv("EMAIL_SUPPORT_CONTACT", None)
+        await self._check_if_using_premium_email_feature(
+            premium_user, email_logo_url, email_support_contact
+        )
+
         event_name = webhook_event.event_message
         recipient_email = webhook_event.user_email
         user_name = webhook_event.user_id
@@ -1266,7 +1283,7 @@ Model Info:
 
         if webhook_event.event == "budget_crossed":
             email_html_content = f"""
-            <img src="{EMAIL_LOGO_URL}" alt="LiteLLM Logo" width="150" height="50" />
+            <img src="{email_logo_url}" alt="LiteLLM Logo" width="150" height="50" />
 
             <p> Hi {user_name}, <br/>
 
@@ -1274,7 +1291,7 @@ Model Info:
 
             API requests will be rejected until either (a) you increase your monthly budget or (b) your monthly usage resets at the beginning of the next calendar month. <br /> <br />
 
-            If you have any questions, please send an email to {EMAIL_SUPPORT_CONTACT} <br /> <br />
+            If you have any questions, please send an email to {email_support_contact} <br /> <br />
 
             Best, <br />
             The LiteLLM team <br />

litellm/llms/openai.py

@@ -157,6 +157,102 @@ class MistralConfig:
                 )
             if param == "seed":
                 optional_params["extra_body"] = {"random_seed": value}
+            if param == "response_format":
+                optional_params["response_format"] = value
+        return optional_params
+
+
+class DeepInfraConfig:
+    """
+    Reference: https://deepinfra.com/docs/advanced/openai_api
+
+    The class `DeepInfra` provides configuration for the DeepInfra's Chat Completions API interface. Below are the parameters:
+    """
+
+    frequency_penalty: Optional[int] = None
+    function_call: Optional[Union[str, dict]] = None
+    functions: Optional[list] = None
+    logit_bias: Optional[dict] = None
+    max_tokens: Optional[int] = None
+    n: Optional[int] = None
+    presence_penalty: Optional[int] = None
+    stop: Optional[Union[str, list]] = None
+    temperature: Optional[int] = None
+    top_p: Optional[int] = None
+    response_format: Optional[dict] = None
+    tools: Optional[list] = None
+    tool_choice: Optional[Union[str, dict]] = None
+
+    def __init__(
+        self,
+        frequency_penalty: Optional[int] = None,
+        function_call: Optional[Union[str, dict]] = None,
+        functions: Optional[list] = None,
+        logit_bias: Optional[dict] = None,
+        max_tokens: Optional[int] = None,
+        n: Optional[int] = None,
+        presence_penalty: Optional[int] = None,
+        stop: Optional[Union[str, list]] = None,
+        temperature: Optional[int] = None,
+        top_p: Optional[int] = None,
+        response_format: Optional[dict] = None,
+        tools: Optional[list] = None,
+        tool_choice: Optional[Union[str, dict]] = None,
+    ) -> None:
+        locals_ = locals().copy()
+        for key, value in locals_.items():
+            if key != "self" and value is not None:
+                setattr(self.__class__, key, value)
+
+    @classmethod
+    def get_config(cls):
+        return {
+            k: v
+            for k, v in cls.__dict__.items()
+            if not k.startswith("__")
+            and not isinstance(
+                v,
+                (
+                    types.FunctionType,
+                    types.BuiltinFunctionType,
+                    classmethod,
+                    staticmethod,
+                ),
+            )
+            and v is not None
+        }
+
+    def get_supported_openai_params(self):
+        return [
+            "stream",
+            "frequency_penalty",
+            "function_call",
+            "functions",
+            "logit_bias",
+            "max_tokens",
+            "n",
+            "presence_penalty",
+            "stop",
+            "temperature",
+            "top_p",
+            "response_format",
+            "tools",
+            "tool_choice",
+        ]
+
+    def map_openai_params(
+        self, non_default_params: dict, optional_params: dict, model: str
+    ):
+        supported_openai_params = self.get_supported_openai_params()
+        for param, value in non_default_params.items():
+            if (
+                param == "temperature"
+                and value == 0
+                and model == "mistralai/Mistral-7B-Instruct-v0.1"
+            ):  # this model does no support temperature == 0
+                value = 0.0001  # close to 0
+            if param in supported_openai_params:
+                optional_params[param] = value
         return optional_params
 
 
@@ -197,6 +293,7 @@ class OpenAIConfig:
     stop: Optional[Union[str, list]] = None
     temperature: Optional[int] = None
     top_p: Optional[int] = None
+    response_format: Optional[dict] = None
 
     def __init__(
         self,
@@ -210,6 +307,7 @@ class OpenAIConfig:
         stop: Optional[Union[str, list]] = None,
         temperature: Optional[int] = None,
         top_p: Optional[int] = None,
+        response_format: Optional[dict] = None,
     ) -> None:
         locals_ = locals().copy()
         for key, value in locals_.items():

litellm/main.py

@@ -294,6 +294,7 @@ async def acompletion(
         "api_version": api_version,
         "api_key": api_key,
         "model_list": model_list,
+        "extra_headers": extra_headers,
         "acompletion": True,  # assuming this is a required parameter
     }
     if custom_llm_provider is None:

litellm/proxy/_experimental/out/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-e85084d25f9ae5e4.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-f960ab1e6d32b002.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-04708d7d4a17c1ee.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-e85084d25f9ae5e4.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/5d93d4a9fa59d72f.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[39712,[\"936\",\"static/chunks/2f6dbc85-052c4579f80d66ae.js\",\"608\",\"static/chunks/608-d128caa3cfe973c1.js\",\"931\",\"static/chunks/app/page-e266cb0126026d40.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/5d93d4a9fa59d72f.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"dYIEEO-62OCgyckEhgBd-\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
+<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-e85084d25f9ae5e4.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-f960ab1e6d32b002.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-04708d7d4a17c1ee.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-e85084d25f9ae5e4.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/5d93d4a9fa59d72f.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[39712,[\"936\",\"static/chunks/2f6dbc85-052c4579f80d66ae.js\",\"608\",\"static/chunks/608-d128caa3cfe973c1.js\",\"931\",\"static/chunks/app/page-76d278f96a0e9768.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/5d93d4a9fa59d72f.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"D_ZUmMtLMPSa4aQQUJtKt\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>

litellm/proxy/_experimental/out/index.txt

@@ -1,7 +1,7 @@
 2:I[77831,[],""]
-3:I[39712,["936","static/chunks/2f6dbc85-052c4579f80d66ae.js","608","static/chunks/608-d128caa3cfe973c1.js","931","static/chunks/app/page-e266cb0126026d40.js"],""]
+3:I[39712,["936","static/chunks/2f6dbc85-052c4579f80d66ae.js","608","static/chunks/608-d128caa3cfe973c1.js","931","static/chunks/app/page-76d278f96a0e9768.js"],""]
 4:I[5613,[],""]
 5:I[31778,[],""]
-0:["dYIEEO-62OCgyckEhgBd-",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/5d93d4a9fa59d72f.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
+0:["D_ZUmMtLMPSa4aQQUJtKt",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/5d93d4a9fa59d72f.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_super_secret_config.yaml

@@ -24,8 +24,8 @@ general_settings:
   database_connection_timeout: 60
   health_check_interval: 300
   ui_access_mode: all
-litellm_settings:
-  json_logs: true
+# litellm_settings:
+#   json_logs: true
 model_list:
 - litellm_params:
     api_base: http://0.0.0.0:8080
@@ -43,5 +43,14 @@ model_list:
     api_version: '2023-05-15'
     model: azure/chatgpt-v-2
   model_name: gpt-3.5-turbo
+- litellm_params:
+    model: anthropic.claude-3-sonnet-20240229-v1:0
+  model_name: bedrock-anthropic-claude-3
+- litellm_params:
+    api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
+    api_key: os.environ/AZURE_API_KEY
+    api_version: '2023-05-15'
+    model: azure/chatgpt-v-2
+  model_name: gpt-3.5-turbo
 router_settings:
   enable_pre_call_checks: true

litellm/proxy/_types.py

@@ -519,7 +519,11 @@ class UpdateUserRequest(GenerateRequestBase):
         return values
 
 
-class NewEndUserRequest(LiteLLMBase):
+class NewCustomerRequest(LiteLLMBase):
+    """
+    Create a new customer, allocate a budget to them
+    """
+
     user_id: str
     alias: Optional[str] = None  # human-friendly alias
     blocked: bool = False  # allow/disallow requests for this end-user
@@ -540,6 +544,33 @@ class NewEndUserRequest(LiteLLMBase):
         return values
 
 
+class UpdateCustomerRequest(LiteLLMBase):
+    """
+    Update a Customer, use this to update customer budgets etc
+
+    """
+
+    user_id: str
+    alias: Optional[str] = None  # human-friendly alias
+    blocked: bool = False  # allow/disallow requests for this end-user
+    max_budget: Optional[float] = None
+    budget_id: Optional[str] = None  # give either a budget_id or max_budget
+    allowed_model_region: Optional[Literal["eu"]] = (
+        None  # require all user requests to use models in this specific region
+    )
+    default_model: Optional[str] = (
+        None  # if no equivalent model in allowed region - default all requests to this model
+    )
+
+
+class DeleteCustomerRequest(LiteLLMBase):
+    """
+    Delete multiple Customers
+    """
+
+    user_ids: List[str]
+
+
 class Member(LiteLLMBase):
     role: Literal["admin", "user"]
     user_id: Optional[str] = None
@@ -928,6 +959,10 @@ class ConfigGeneralSettings(LiteLLMBase):
     allowed_routes: Optional[List] = Field(
         None, description="Proxy API Endpoints you want users to be able to access"
     )
+    enable_public_model_hub: bool = Field(
+        default=False,
+        description="Public model hub for users to see what models they have access to, supported openai params, etc.",
+    )
 
 
 class ConfigYAML(LiteLLMBase):
@@ -984,10 +1019,6 @@ class LiteLLM_VerificationToken(LiteLLMBase):
 
     org_id: Optional[str] = None  # org id for a given key
 
-    # hidden params used for parallel request limiting, not required to create a token
-    user_id_rate_limits: Optional[dict] = None
-    team_id_rate_limits: Optional[dict] = None
-
     class Config:
         protected_namespaces = ()
 
@@ -1158,3 +1189,32 @@ class WebhookEvent(CallInfo):
 class SpecialModelNames(enum.Enum):
     all_team_models = "all-team-models"
     all_proxy_models = "all-proxy-models"
+
+
+class InvitationNew(LiteLLMBase):
+    user_id: str
+
+
+class InvitationUpdate(LiteLLMBase):
+    invitation_id: str
+    is_accepted: bool
+
+
+class InvitationDelete(LiteLLMBase):
+    invitation_id: str
+
+
+class InvitationModel(LiteLLMBase):
+    id: str
+    user_id: str
+    is_accepted: bool
+    accepted_at: Optional[datetime]
+    expires_at: datetime
+    created_at: datetime
+    created_by: str
+    updated_at: datetime
+    updated_by: str
+
+class ConfigFieldInfo(LiteLLMBase):
+    field_name: str
+    field_value: Any

litellm/proxy/hooks/parallel_request_limiter.py

@@ -164,8 +164,9 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
         # check if REQUEST ALLOWED for user_id
         user_id = user_api_key_dict.user_id
         if user_id is not None:
-            _user_id_rate_limits = user_api_key_dict.user_id_rate_limits
-
+            _user_id_rate_limits = await self.user_api_key_cache.async_get_cache(
+                key=user_id
+            )
             # get user tpm/rpm limits
             if _user_id_rate_limits is not None and isinstance(
                 _user_id_rate_limits, dict
@@ -196,13 +197,8 @@ class _PROXY_MaxParallelRequestsHandler(CustomLogger):
         ## get team tpm/rpm limits
         team_id = user_api_key_dict.team_id
         if team_id is not None:
-            team_tpm_limit = getattr(user_api_key_dict, "team_tpm_limit", sys.maxsize)
-
-            if team_tpm_limit is None:
-                team_tpm_limit = sys.maxsize
-            team_rpm_limit = getattr(user_api_key_dict, "team_rpm_limit", sys.maxsize)
-            if team_rpm_limit is None:
-                team_rpm_limit = sys.maxsize
+            team_tpm_limit = user_api_key_dict.team_tpm_limit
+            team_rpm_limit = user_api_key_dict.team_rpm_limit
 
             if team_tpm_limit is None:
                 team_tpm_limit = sys.maxsize

litellm/proxy/hooks/tpm_rpm_limiter.py

@@ -1,379 +0,0 @@
-# What is this?
-## Checks TPM/RPM Limits for a key/user/team on the proxy
-## Works with Redis - if given
-
-from typing import Optional, Literal
-import litellm, traceback, sys
-from litellm.caching import DualCache, RedisCache
-from litellm.proxy._types import (
-    UserAPIKeyAuth,
-    LiteLLM_VerificationTokenView,
-    LiteLLM_UserTable,
-    LiteLLM_TeamTable,
-)
-from litellm.integrations.custom_logger import CustomLogger
-from fastapi import HTTPException
-from litellm._logging import verbose_proxy_logger
-from litellm import ModelResponse
-from datetime import datetime
-
-
-class _PROXY_MaxTPMRPMLimiter(CustomLogger):
-    user_api_key_cache = None
-
-    # Class variables or attributes
-    def __init__(self, internal_cache: Optional[DualCache]):
-        if internal_cache is None:
-            self.internal_cache = DualCache()
-        else:
-            self.internal_cache = internal_cache
-
-    def print_verbose(self, print_statement):
-        try:
-            verbose_proxy_logger.debug(print_statement)
-            if litellm.set_verbose:
-                print(print_statement)  # noqa
-        except:
-            pass
-
-            ## check if admin has set tpm/rpm limits for this key/user/team
-
-    def _check_limits_set(
-        self,
-        user_api_key_cache: DualCache,
-        key: Optional[str],
-        user_id: Optional[str],
-        team_id: Optional[str],
-    ) -> bool:
-        ## key
-        if key is not None:
-            key_val = user_api_key_cache.get_cache(key=key)
-            if isinstance(key_val, dict):
-                key_val = LiteLLM_VerificationTokenView(**key_val)
-
-            if isinstance(key_val, LiteLLM_VerificationTokenView):
-                user_api_key_tpm_limit = key_val.tpm_limit
-
-                user_api_key_rpm_limit = key_val.rpm_limit
-
-                if (
-                    user_api_key_tpm_limit is not None
-                    or user_api_key_rpm_limit is not None
-                ):
-                    return True
-
-        ## team
-        if team_id is not None:
-            team_val = user_api_key_cache.get_cache(key=team_id)
-            if isinstance(team_val, dict):
-                team_val = LiteLLM_TeamTable(**team_val)
-
-            if isinstance(team_val, LiteLLM_TeamTable):
-                team_tpm_limit = team_val.tpm_limit
-
-                team_rpm_limit = team_val.rpm_limit
-
-                if team_tpm_limit is not None or team_rpm_limit is not None:
-                    return True
-
-        ## user
-        if user_id is not None:
-            user_val = user_api_key_cache.get_cache(key=user_id)
-            if isinstance(user_val, dict):
-                user_val = LiteLLM_UserTable(**user_val)
-
-            if isinstance(user_val, LiteLLM_UserTable):
-                user_tpm_limit = user_val.tpm_limit
-
-                user_rpm_limit = user_val.rpm_limit
-
-                if user_tpm_limit is not None or user_rpm_limit is not None:
-                    return True
-        return False
-
-    async def check_key_in_limits(
-        self,
-        user_api_key_dict: UserAPIKeyAuth,
-        current_minute_dict: dict,
-        tpm_limit: int,
-        rpm_limit: int,
-        request_count_api_key: str,
-        type: Literal["key", "user", "team"],
-    ):
-
-        if type == "key" and user_api_key_dict.api_key is not None:
-            current = current_minute_dict["key"].get(user_api_key_dict.api_key, None)
-        elif type == "user" and user_api_key_dict.user_id is not None:
-            current = current_minute_dict["user"].get(user_api_key_dict.user_id, None)
-        elif type == "team" and user_api_key_dict.team_id is not None:
-            current = current_minute_dict["team"].get(user_api_key_dict.team_id, None)
-        else:
-            return
-        if current is None:
-            if tpm_limit == 0 or rpm_limit == 0:
-                # base case
-                raise HTTPException(
-                    status_code=429, detail="Max tpm/rpm limit reached."
-                )
-        elif current["current_tpm"] < tpm_limit and current["current_rpm"] < rpm_limit:
-            pass
-        else:
-            raise HTTPException(status_code=429, detail="Max tpm/rpm limit reached.")
-
-    async def async_pre_call_hook(
-        self,
-        user_api_key_dict: UserAPIKeyAuth,
-        cache: DualCache,
-        data: dict,
-        call_type: str,
-    ):
-        self.print_verbose(
-            f"Inside Max TPM/RPM Limiter Pre-Call Hook - {user_api_key_dict}"
-        )
-        api_key = user_api_key_dict.api_key
-        # check if REQUEST ALLOWED for user_id
-        user_id = user_api_key_dict.user_id
-        ## get team tpm/rpm limits
-        team_id = user_api_key_dict.team_id
-
-        self.user_api_key_cache = cache
-
-        _set_limits = self._check_limits_set(
-            user_api_key_cache=cache, key=api_key, user_id=user_id, team_id=team_id
-        )
-
-        self.print_verbose(f"_set_limits: {_set_limits}")
-
-        if _set_limits == False:
-            return
-
-        # ------------
-        # Setup values
-        # ------------
-
-        current_date = datetime.now().strftime("%Y-%m-%d")
-        current_hour = datetime.now().strftime("%H")
-        current_minute = datetime.now().strftime("%M")
-        precise_minute = f"{current_date}-{current_hour}-{current_minute}"
-        cache_key = "usage:{}".format(precise_minute)
-        current_minute_dict = await self.internal_cache.async_get_cache(
-            key=cache_key
-        )  # {"usage:{curr_minute}": {"key": {<api_key>: {"current_requests": 1, "current_tpm": 1, "current_rpm": 10}}}}
-
-        if current_minute_dict is None:
-            current_minute_dict = {"key": {}, "user": {}, "team": {}}
-
-        if api_key is not None:
-            tpm_limit = getattr(user_api_key_dict, "tpm_limit", sys.maxsize)
-            if tpm_limit is None:
-                tpm_limit = sys.maxsize
-            rpm_limit = getattr(user_api_key_dict, "rpm_limit", sys.maxsize)
-            if rpm_limit is None:
-                rpm_limit = sys.maxsize
-            request_count_api_key = f"{api_key}::{precise_minute}::request_count"
-            await self.check_key_in_limits(
-                user_api_key_dict=user_api_key_dict,
-                current_minute_dict=current_minute_dict,
-                request_count_api_key=request_count_api_key,
-                tpm_limit=tpm_limit,
-                rpm_limit=rpm_limit,
-                type="key",
-            )
-
-        if user_id is not None:
-            _user_id_rate_limits = user_api_key_dict.user_id_rate_limits
-
-            # get user tpm/rpm limits
-            if _user_id_rate_limits is not None and isinstance(
-                _user_id_rate_limits, dict
-            ):
-                user_tpm_limit = _user_id_rate_limits.get("tpm_limit", None)
-                user_rpm_limit = _user_id_rate_limits.get("rpm_limit", None)
-                if user_tpm_limit is None:
-                    user_tpm_limit = sys.maxsize
-                if user_rpm_limit is None:
-                    user_rpm_limit = sys.maxsize
-
-                # now do the same tpm/rpm checks
-                request_count_api_key = f"{user_id}::{precise_minute}::request_count"
-                # print(f"Checking if {request_count_api_key} is allowed to make request for minute {precise_minute}")
-                await self.check_key_in_limits(
-                    user_api_key_dict=user_api_key_dict,
-                    current_minute_dict=current_minute_dict,
-                    request_count_api_key=request_count_api_key,
-                    tpm_limit=user_tpm_limit,
-                    rpm_limit=user_rpm_limit,
-                    type="user",
-                )
-
-        # TEAM RATE LIMITS
-        if team_id is not None:
-            team_tpm_limit = getattr(user_api_key_dict, "team_tpm_limit", sys.maxsize)
-
-            if team_tpm_limit is None:
-                team_tpm_limit = sys.maxsize
-            team_rpm_limit = getattr(user_api_key_dict, "team_rpm_limit", sys.maxsize)
-            if team_rpm_limit is None:
-                team_rpm_limit = sys.maxsize
-
-            if team_tpm_limit is None:
-                team_tpm_limit = sys.maxsize
-            if team_rpm_limit is None:
-                team_rpm_limit = sys.maxsize
-
-            # now do the same tpm/rpm checks
-            request_count_api_key = f"{team_id}::{precise_minute}::request_count"
-
-            # print(f"Checking if {request_count_api_key} is allowed to make request for minute {precise_minute}")
-            await self.check_key_in_limits(
-                user_api_key_dict=user_api_key_dict,
-                current_minute_dict=current_minute_dict,
-                request_count_api_key=request_count_api_key,
-                tpm_limit=team_tpm_limit,
-                rpm_limit=team_rpm_limit,
-                type="team",
-            )
-
-        return
-
-    async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
-        try:
-            self.print_verbose(f"INSIDE TPM RPM Limiter ASYNC SUCCESS LOGGING")
-
-            user_api_key = kwargs["litellm_params"]["metadata"]["user_api_key"]
-            user_api_key_user_id = kwargs["litellm_params"]["metadata"].get(
-                "user_api_key_user_id", None
-            )
-            user_api_key_team_id = kwargs["litellm_params"]["metadata"].get(
-                "user_api_key_team_id", None
-            )
-            _limits_set = self._check_limits_set(
-                user_api_key_cache=self.user_api_key_cache,
-                key=user_api_key,
-                user_id=user_api_key_user_id,
-                team_id=user_api_key_team_id,
-            )
-
-            if _limits_set == False:  # don't waste cache calls if no tpm/rpm limits set
-                return
-
-            # ------------
-            # Setup values
-            # ------------
-
-            current_date = datetime.now().strftime("%Y-%m-%d")
-            current_hour = datetime.now().strftime("%H")
-            current_minute = datetime.now().strftime("%M")
-            precise_minute = f"{current_date}-{current_hour}-{current_minute}"
-
-            total_tokens = 0
-
-            if isinstance(response_obj, ModelResponse):
-                total_tokens = response_obj.usage.total_tokens
-
-            """
-            - get value from redis
-            - increment requests + 1
-            - increment tpm + 1
-            - increment rpm + 1
-            - update value in-memory + redis
-            """
-            cache_key = "usage:{}".format(precise_minute)
-            if (
-                self.internal_cache.redis_cache is not None
-            ):  # get straight from redis if possible
-                current_minute_dict = (
-                    await self.internal_cache.redis_cache.async_get_cache(
-                        key=cache_key,
-                    )
-                )  # {"usage:{current_minute}": {"key": {}, "team": {}, "user": {}}}
-            else:
-                current_minute_dict = await self.internal_cache.async_get_cache(
-                    key=cache_key,
-                )
-
-            if current_minute_dict is None:
-                current_minute_dict = {"key": {}, "user": {}, "team": {}}
-
-            _cache_updated = False  # check if a cache update is required. prevent unnecessary rewrites.
-
-            # ------------
-            # Update usage - API Key
-            # ------------
-
-            if user_api_key is not None:
-                _cache_updated = True
-                ## API KEY ##
-                if user_api_key in current_minute_dict["key"]:
-                    current_key_usage = current_minute_dict["key"][user_api_key]
-                    new_val = {
-                        "current_tpm": current_key_usage["current_tpm"] + total_tokens,
-                        "current_rpm": current_key_usage["current_rpm"] + 1,
-                    }
-                else:
-                    new_val = {
-                        "current_tpm": total_tokens,
-                        "current_rpm": 1,
-                    }
-
-                current_minute_dict["key"][user_api_key] = new_val
-
-                self.print_verbose(
-                    f"updated_value in success call: {new_val}, precise_minute: {precise_minute}"
-                )
-
-            # ------------
-            # Update usage - User
-            # ------------
-            if user_api_key_user_id is not None:
-                _cache_updated = True
-                total_tokens = 0
-
-                if isinstance(response_obj, ModelResponse):
-                    total_tokens = response_obj.usage.total_tokens
-
-                if user_api_key_user_id in current_minute_dict["key"]:
-                    current_key_usage = current_minute_dict["key"][user_api_key_user_id]
-                    new_val = {
-                        "current_tpm": current_key_usage["current_tpm"] + total_tokens,
-                        "current_rpm": current_key_usage["current_rpm"] + 1,
-                    }
-                else:
-                    new_val = {
-                        "current_tpm": total_tokens,
-                        "current_rpm": 1,
-                    }
-
-                current_minute_dict["user"][user_api_key_user_id] = new_val
-
-            # ------------
-            # Update usage - Team
-            # ------------
-            if user_api_key_team_id is not None:
-                _cache_updated = True
-                total_tokens = 0
-
-                if isinstance(response_obj, ModelResponse):
-                    total_tokens = response_obj.usage.total_tokens
-
-                if user_api_key_team_id in current_minute_dict["key"]:
-                    current_key_usage = current_minute_dict["key"][user_api_key_team_id]
-                    new_val = {
-                        "current_tpm": current_key_usage["current_tpm"] + total_tokens,
-                        "current_rpm": current_key_usage["current_rpm"] + 1,
-                    }
-                else:
-                    new_val = {
-                        "current_tpm": total_tokens,
-                        "current_rpm": 1,
-                    }
-
-                current_minute_dict["team"][user_api_key_team_id] = new_val
-
-            if _cache_updated == True:
-                await self.internal_cache.async_set_cache(
-                    key=cache_key, value=current_minute_dict
-                )
-
-        except Exception as e:
-            self.print_verbose("{}\n{}".format(e, traceback.format_exc()))  # noqa

litellm/proxy/proxy_server.py

@@ -397,6 +397,7 @@ def _get_pydantic_json_dict(pydantic_obj: BaseModel) -> dict:
 
 def get_custom_headers(
     *,
+    user_api_key_dict: UserAPIKeyAuth,
     model_id: Optional[str] = None,
     cache_key: Optional[str] = None,
     api_base: Optional[str] = None,
@@ -410,6 +411,8 @@ def get_custom_headers(
         "x-litellm-model-api-base": api_base,
         "x-litellm-version": version,
         "x-litellm-model-region": model_region,
+        "x-litellm-key-tpm-limit": str(user_api_key_dict.tpm_limit),
+        "x-litellm-key-rpm-limit": str(user_api_key_dict.rpm_limit),
     }
     try:
         return {
@@ -2787,6 +2790,13 @@ class ProxyConfig:
             model.model_info["id"] = _id
             model.model_info["db_model"] = True
 
+        if premium_user is True:
+            # seeing "created_at", "updated_at", "created_by", "updated_by" is a LiteLLM Enterprise Feature
+            model.model_info["created_at"] = getattr(model, "created_at", None)
+            model.model_info["updated_at"] = getattr(model, "updated_at", None)
+            model.model_info["created_by"] = getattr(model, "created_by", None)
+            model.model_info["updated_by"] = getattr(model, "updated_by", None)
+
         if model.model_info is not None and isinstance(model.model_info, dict):
             if "id" not in model.model_info:
                 model.model_info["id"] = model.model_id
@@ -3072,10 +3082,9 @@ class ProxyConfig:
 
         try:
             if master_key is None or not isinstance(master_key, str):
-                raise Exception(
+                raise ValueError(
                     f"Master key is not initialized or formatted. master_key={master_key}"
                 )
-            verbose_proxy_logger.debug(f"llm_router: {llm_router}")
             new_models = await prisma_client.db.litellm_proxymodeltable.find_many()
             # update llm router
             await self._update_llm_router(
@@ -4059,6 +4068,7 @@ async def chat_completion(
             "stream" in data and data["stream"] == True
         ):  # use generate_responses to stream responses
             custom_headers = get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
@@ -4078,6 +4088,7 @@ async def chat_completion(
 
         fastapi_response.headers.update(
             get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
@@ -4298,6 +4309,7 @@ async def completion(
             "stream" in data and data["stream"] == True
         ):  # use generate_responses to stream responses
             custom_headers = get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
@@ -4316,6 +4328,7 @@ async def completion(
             )
         fastapi_response.headers.update(
             get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
@@ -4565,6 +4578,7 @@ async def embeddings(
 
         fastapi_response.headers.update(
             get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
@@ -4748,6 +4762,7 @@ async def image_generation(
 
         fastapi_response.headers.update(
             get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
@@ -4949,6 +4964,7 @@ async def audio_transcriptions(
 
         fastapi_response.headers.update(
             get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
@@ -5132,6 +5148,7 @@ async def moderations(
 
         fastapi_response.headers.update(
             get_custom_headers(
+                user_api_key_dict=user_api_key_dict,
                 model_id=model_id,
                 cache_key=cache_key,
                 api_base=api_base,
@@ -6083,7 +6100,7 @@ async def get_global_activity_model(
 
         sql_query = """
         SELECT
-            model,
+            model_group AS model,
             date_trunc('day', "startTime") AS date,
             COUNT(*) AS api_requests,
             SUM(total_tokens) AS total_tokens
@@ -7120,13 +7137,15 @@ async def global_predict_spend_logs(request: Request):
 #### INTERNAL USER MANAGEMENT ####
 @router.post(
     "/user/new",
-    tags=["user management"],
+    tags=["Internal User management"],
     dependencies=[Depends(user_api_key_auth)],
     response_model=NewUserResponse,
 )
 async def new_user(data: NewUserRequest):
     """
-    Use this to create a new user with a budget. This creates a new user and generates a new api key for the new user. The new api key is returned.
+    Use this to create a new INTERNAL user with a budget.
+    Internal Users can access LiteLLM Admin UI to make keys, request access to models.
+    This creates a new user and generates a new api key for the new user. The new api key is returned.
 
     Returns user id, budget + new key.
 
@@ -7197,7 +7216,9 @@ async def new_user(data: NewUserRequest):
 
 
 @router.post(
-    "/user/auth", tags=["user management"], dependencies=[Depends(user_api_key_auth)]
+    "/user/auth",
+    tags=["Internal User management"],
+    dependencies=[Depends(user_api_key_auth)],
 )
 async def user_auth(request: Request):
     """
@@ -7263,7 +7284,9 @@ async def user_auth(request: Request):
 
 
 @router.get(
-    "/user/info", tags=["user management"], dependencies=[Depends(user_api_key_auth)]
+    "/user/info",
+    tags=["Internal User management"],
+    dependencies=[Depends(user_api_key_auth)],
 )
 async def user_info(
     user_id: Optional[str] = fastapi.Query(
@@ -7435,7 +7458,9 @@ async def user_info(
 
 
 @router.post(
-    "/user/update", tags=["user management"], dependencies=[Depends(user_api_key_auth)]
+    "/user/update",
+    tags=["Internal User management"],
+    dependencies=[Depends(user_api_key_auth)],
 )
 async def user_update(data: UpdateUserRequest):
     """
@@ -7529,7 +7554,7 @@ async def user_update(data: UpdateUserRequest):
 
 @router.post(
     "/user/request_model",
-    tags=["user management"],
+    tags=["Internal User management"],
     dependencies=[Depends(user_api_key_auth)],
 )
 async def user_request_model(request: Request):
@@ -7582,7 +7607,7 @@ async def user_request_model(request: Request):
 
 @router.get(
     "/user/get_requests",
-    tags=["user management"],
+    tags=["Internal User management"],
     dependencies=[Depends(user_api_key_auth)],
 )
 async def user_get_requests():
@@ -7624,7 +7649,7 @@ async def user_get_requests():
 
 @router.get(
     "/user/get_users",
-    tags=["user management"],
+    tags=["Internal User management"],
     dependencies=[Depends(user_api_key_auth)],
 )
 async def get_users(
@@ -7661,7 +7686,13 @@ async def get_users(
 
 @router.post(
     "/end_user/block",
-    tags=["End User Management"],
+    tags=["Customer Management"],
+    dependencies=[Depends(user_api_key_auth)],
+    include_in_schema=False,
+)
+@router.post(
+    "/customer/block",
+    tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
 )
 async def block_user(data: BlockUsers):
@@ -7704,9 +7735,15 @@ async def block_user(data: BlockUsers):
 
 @router.post(
     "/end_user/unblock",
-    tags=["End User Management"],
+    tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
 )
+@router.post(
+    "/customer/unblock",
+    tags=["Customer Management"],
+    dependencies=[Depends(user_api_key_auth)],
+    include_in_schema=False,
+)
 async def unblock_user(data: BlockUsers):
     """
     [BETA] Unblock calls with this user id
@@ -7751,35 +7788,36 @@ async def unblock_user(data: BlockUsers):
 
 @router.post(
     "/end_user/new",
-    tags=["End User Management"],
+    tags=["Customer Management"],
+    include_in_schema=False,
+    dependencies=[Depends(user_api_key_auth)],
+)
+@router.post(
+    "/customer/new",
+    tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
 )
 async def new_end_user(
-    data: NewEndUserRequest,
+    data: NewCustomerRequest,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
 ):
     """
-    [TODO] Needs to be implemented.
-
-    Allow creating a new end-user 
+    Allow creating a new Customer 
+    NOTE: This used to be called `/end_user/new`, we will still be maintaining compatibility for /end_user/XXX for these endpoints
 
     - Allow specifying allowed regions 
     - Allow specifying default model
 
     Example curl:
     ```
-    curl --location 'http://0.0.0.0:4000/end_user/new' \
+    curl --location 'http://0.0.0.0:4000/customer/new' \
         --header 'Authorization: Bearer sk-1234' \
         --header 'Content-Type: application/json' \
         --data '{
-            "end_user_id" : "ishaan-jaff-3", <- specific customer
-            
-            "allowed_region": "eu" <- set region for models        
-
-                    + 
-
+            "user_id" : "ishaan-jaff-3",
+            "allowed_region": "eu",
+            "budget_id": "free_tier",
             "default_model": "azure/gpt-3.5-turbo-eu" <- all calls from this user, use this model? 
-
         }'
 
         # return end-user object
@@ -7802,56 +7840,88 @@ async def new_end_user(
             status_code=500,
             detail={"error": CommonProxyErrors.db_not_connected_error.value},
         )
+    try:
 
-    ## VALIDATION ##
-    if data.default_model is not None:
-        if llm_router is None:
-            raise HTTPException(
-                status_code=422, detail={"error": CommonProxyErrors.no_llm_router.value}
-            )
-        elif data.default_model not in llm_router.get_model_names():
-            raise HTTPException(
-                status_code=422,
-                detail={
-                    "error": "Default Model not on proxy. Configure via `/model/new` or config.yaml. Default_model={}, proxy_model_names={}".format(
-                        data.default_model, set(llm_router.get_model_names())
-                    )
-                },
+        ## VALIDATION ##
+        if data.default_model is not None:
+            if llm_router is None:
+                raise HTTPException(
+                    status_code=422,
+                    detail={"error": CommonProxyErrors.no_llm_router.value},
+                )
+            elif data.default_model not in llm_router.get_model_names():
+                raise HTTPException(
+                    status_code=422,
+                    detail={
+                        "error": "Default Model not on proxy. Configure via `/model/new` or config.yaml. Default_model={}, proxy_model_names={}".format(
+                            data.default_model, set(llm_router.get_model_names())
+                        )
+                    },
+                )
+
+        new_end_user_obj: Dict = {}
+
+        ## CREATE BUDGET ## if set
+        if data.max_budget is not None:
+            budget_record = await prisma_client.db.litellm_budgettable.create(
+                data={
+                    "max_budget": data.max_budget,
+                    "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,  # type: ignore
+                    "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+                }
             )
 
-    new_end_user_obj: Dict = {}
+            new_end_user_obj["budget_id"] = budget_record.budget_id
+        elif data.budget_id is not None:
+            new_end_user_obj["budget_id"] = data.budget_id
 
-    ## CREATE BUDGET ## if set
-    if data.max_budget is not None:
-        budget_record = await prisma_client.db.litellm_budgettable.create(
-            data={
-                "max_budget": data.max_budget,
-                "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,  # type: ignore
-                "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
-            }
+        _user_data = data.dict(exclude_none=True)
+
+        for k, v in _user_data.items():
+            if k != "max_budget" and k != "budget_id":
+                new_end_user_obj[k] = v
+
+        ## WRITE TO DB ##
+        end_user_record = await prisma_client.db.litellm_endusertable.create(
+            data=new_end_user_obj  # type: ignore
         )
 
-        new_end_user_obj["budget_id"] = budget_record.budget_id
-    elif data.budget_id is not None:
-        new_end_user_obj["budget_id"] = data.budget_id
+        return end_user_record
+    except Exception as e:
+        if "Unique constraint failed on the fields: (`user_id`)" in str(e):
+            raise ProxyException(
+                message=f"Customer already exists, passed user_id={data.user_id}. Please pass a new user_id.",
+                type="bad_request",
+                code=400,
+                param="user_id",
+            )
 
-    _user_data = data.dict(exclude_none=True)
-
-    for k, v in _user_data.items():
-        if k != "max_budget" and k != "budget_id":
-            new_end_user_obj[k] = v
-
-    ## WRITE TO DB ##
-    end_user_record = await prisma_client.db.litellm_endusertable.create(
-        data=new_end_user_obj  # type: ignore
-    )
-
-    return end_user_record
+        if isinstance(e, HTTPException):
+            raise ProxyException(
+                message=getattr(e, "detail", f"Internal Server Error({str(e)})"),
+                type="internal_error",
+                param=getattr(e, "param", "None"),
+                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
+            )
+        elif isinstance(e, ProxyException):
+            raise e
+        raise ProxyException(
+            message="Internal Server Error, " + str(e),
+            type="internal_error",
+            param=getattr(e, "param", "None"),
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
 
 
+@router.get(
+    "/customer/info",
+    tags=["Customer Management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
 @router.get(
     "/end_user/info",
-    tags=["End User Management"],
+    tags=["Customer Management"],
+    include_in_schema=False,
     dependencies=[Depends(user_api_key_auth)],
 )
 async def end_user_info(
@@ -7875,26 +7945,174 @@ async def end_user_info(
 
 
 @router.post(
-    "/end_user/update",
-    tags=["End User Management"],
+    "/customer/update",
+    tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
 )
-async def update_end_user():
+@router.post(
+    "/end_user/update",
+    tags=["Customer Management"],
+    include_in_schema=False,
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def update_end_user(
+    data: UpdateCustomerRequest,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     """
-    [TODO] Needs to be implemented.
+    Example curl 
+
+    ```
+    curl --location 'http://0.0.0.0:4000/customer/update' \
+    --header 'Authorization: Bearer sk-1234' \
+    --header 'Content-Type: application/json' \
+    --data '{
+        "user_id": "test-litellm-user-4",
+        "budget_id": "paid_tier"
+    }'
+
+    See below for all params 
+    ```
     """
+
+    global prisma_client
+    try:
+        data_json: dict = data.json()
+        # get the row from db
+        if prisma_client is None:
+            raise Exception("Not connected to DB!")
+
+        # get non default values for key
+        non_default_values = {}
+        for k, v in data_json.items():
+            if v is not None and v not in (
+                [],
+                {},
+                0,
+            ):  # models default to [], spend defaults to 0, we should not reset these values
+                non_default_values[k] = v
+
+        ## ADD USER, IF NEW ##
+        verbose_proxy_logger.debug("/customer/update: Received data = %s", data)
+        if data.user_id is not None and len(data.user_id) > 0:
+            non_default_values["user_id"] = data.user_id  # type: ignore
+            verbose_proxy_logger.debug("In update customer, user_id condition block.")
+            response = await prisma_client.db.litellm_endusertable.update(
+                where={"user_id": data.user_id}, data=non_default_values  # type: ignore
+            )
+            if response is None:
+                raise ValueError(
+                    f"Failed updating customer data. User ID does not exist passed user_id={data.user_id}"
+                )
+            verbose_proxy_logger.debug(
+                f"received response from updating prisma client. response={response}"
+            )
+            return response
+        else:
+            raise ValueError(f"user_id is required, passed user_id = {data.user_id}")
+
+        # update based on remaining passed in values
+    except Exception as e:
+        traceback.print_exc()
+        if isinstance(e, HTTPException):
+            raise ProxyException(
+                message=getattr(e, "detail", f"Internal Server Error({str(e)})"),
+                type="internal_error",
+                param=getattr(e, "param", "None"),
+                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
+            )
+        elif isinstance(e, ProxyException):
+            raise e
+        raise ProxyException(
+            message="Internal Server Error, " + str(e),
+            type="internal_error",
+            param=getattr(e, "param", "None"),
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
     pass
 
 
 @router.post(
-    "/end_user/delete",
-    tags=["End User Management"],
+    "/customer/delete",
+    tags=["Customer Management"],
     dependencies=[Depends(user_api_key_auth)],
 )
-async def delete_end_user():
+@router.post(
+    "/end_user/delete",
+    tags=["Customer Management"],
+    include_in_schema=False,
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def delete_end_user(
+    data: DeleteCustomerRequest,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     """
-    [TODO] Needs to be implemented.
+    Example curl 
+
+    ```
+    curl --location 'http://0.0.0.0:4000/customer/delete' \
+        --header 'Authorization: Bearer sk-1234' \
+        --header 'Content-Type: application/json' \
+        --data '{
+            "user_ids" :["ishaan-jaff-5"]
+    }'
+
+    See below for all params 
+    ```
     """
+    global prisma_client
+
+    try:
+        if prisma_client is None:
+            raise Exception("Not connected to DB!")
+
+        verbose_proxy_logger.debug("/customer/delete: Received data = %s", data)
+        if (
+            data.user_ids is not None
+            and isinstance(data.user_ids, list)
+            and len(data.user_ids) > 0
+        ):
+            response = await prisma_client.db.litellm_endusertable.delete_many(
+                where={"user_id": {"in": data.user_ids}}
+            )
+            if response is None:
+                raise ValueError(
+                    f"Failed deleting customer data. User ID does not exist passed user_id={data.user_ids}"
+                )
+            if response != len(data.user_ids):
+                raise ValueError(
+                    f"Failed deleting all customer data. User ID does not exist passed user_id={data.user_ids}. Deleted {response} customers, passed {len(data.user_ids)} customers"
+                )
+            verbose_proxy_logger.debug(
+                f"received response from updating prisma client. response={response}"
+            )
+            return {
+                "deleted_customers": response,
+                "message": "Successfully deleted customers with ids: "
+                + str(data.user_ids),
+            }
+        else:
+            raise ValueError(f"user_id is required, passed user_id = {data.user_ids}")
+
+        # update based on remaining passed in values
+    except Exception as e:
+        traceback.print_exc()
+        if isinstance(e, HTTPException):
+            raise ProxyException(
+                message=getattr(e, "detail", f"Internal Server Error({str(e)})"),
+                type="internal_error",
+                param=getattr(e, "param", "None"),
+                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
+            )
+        elif isinstance(e, ProxyException):
+            raise e
+        raise ProxyException(
+            message="Internal Server Error, " + str(e),
+            type="internal_error",
+            param=getattr(e, "param", "None"),
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
     pass
 
 
@@ -10824,6 +11042,236 @@ async def auth_callback(request: Request):
     return RedirectResponse(url=litellm_dashboard_ui)
 
 
+#### INVITATION MANAGEMENT ####
+
+
+@router.post(
+    "/invitation/new",
+    tags=["Invite Links"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=InvitationModel,
+)
+async def new_invitation(
+    data: InvitationNew, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)
+):
+    """
+    Allow admin to create invite links, to onboard new users to Admin UI.
+
+    ```
+    curl -X POST 'http://localhost:4000/invitation/new' \
+        -H 'Content-Type: application/json' \
+        -D '{
+            "user_id": "1234" // 👈 id of user in 'LiteLLM_UserTable'
+        }'
+    ```
+    """
+    global prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": CommonProxyErrors.db_not_connected_error.value},
+        )
+
+    if user_api_key_dict.user_role != "proxy_admin":
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": "{}, your role={}".format(
+                    CommonProxyErrors.not_allowed_access.value,
+                    user_api_key_dict.user_role,
+                )
+            },
+        )
+
+    current_time = litellm.utils.get_utc_datetime()
+    expires_at = current_time + timedelta(days=7)
+
+    try:
+        response = await prisma_client.db.litellm_invitationlink.create(
+            data={
+                "user_id": data.user_id,
+                "created_at": current_time,
+                "expires_at": expires_at,
+                "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+                "updated_at": current_time,
+                "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+            }  # type: ignore
+        )
+    except Exception as e:
+        if "Foreign key constraint failed on the field" in str(e):
+            raise HTTPException(
+                status_code=400,
+                detail={
+                    "error": "User id does not exist in 'LiteLLM_UserTable'. Fix this by creating user via `/user/new`."
+                },
+            )
+    return response
+
+
+@router.get(
+    "/invitation/info",
+    tags=["Invite Links"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=InvitationModel,
+)
+async def invitation_info(
+    invitation_id: str, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth)
+):
+    """
+    Allow admin to create invite links, to onboard new users to Admin UI.
+
+    ```
+    curl -X POST 'http://localhost:4000/invitation/new' \
+        -H 'Content-Type: application/json' \
+        -D '{
+            "user_id": "1234" // 👈 id of user in 'LiteLLM_UserTable'
+        }'
+    ```
+    """
+    global prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": CommonProxyErrors.db_not_connected_error.value},
+        )
+
+    if user_api_key_dict.user_role != "proxy_admin":
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": "{}, your role={}".format(
+                    CommonProxyErrors.not_allowed_access.value,
+                    user_api_key_dict.user_role,
+                )
+            },
+        )
+
+    response = await prisma_client.db.litellm_invitationlink.find_unique(
+        where={"id": invitation_id}
+    )
+
+    if response is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": "Invitation id does not exist in the database."},
+        )
+    return response
+
+
+@router.post(
+    "/invitation/update",
+    tags=["Invite Links"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=InvitationModel,
+)
+async def invitation_update(
+    data: InvitationUpdate,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Update when invitation is accepted
+    
+    ```
+    curl -X POST 'http://localhost:4000/invitation/update' \
+        -H 'Content-Type: application/json' \
+        -D '{
+            "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable'
+            "is_accepted": True // when invitation is accepted
+        }'
+    ```
+    """
+    global prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": CommonProxyErrors.db_not_connected_error.value},
+        )
+
+    if user_api_key_dict.user_id is None:
+        raise HTTPException(
+            status_code=500,
+            detail={
+                "error": "Unable to identify user id. Received={}".format(
+                    user_api_key_dict.user_id
+                )
+            },
+        )
+
+    current_time = litellm.utils.get_utc_datetime()
+    response = await prisma_client.db.litellm_invitationlink.update(
+        where={"id": data.invitation_id},
+        data={
+            "id": data.invitation_id,
+            "is_accepted": data.is_accepted,
+            "accepted_at": current_time,
+            "updated_at": current_time,
+            "updated_by": user_api_key_dict.user_id,  # type: ignore
+        },
+    )
+
+    if response is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": "Invitation id does not exist in the database."},
+        )
+    return response
+
+
+@router.post(
+    "/invitation/delete",
+    tags=["Invite Links"],
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=InvitationModel,
+)
+async def invitation_delete(
+    data: InvitationDelete,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Delete invitation link
+    
+    ```
+    curl -X POST 'http://localhost:4000/invitation/delete' \
+        -H 'Content-Type: application/json' \
+        -D '{
+            "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable'
+        }'
+    ```
+    """
+    global prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": CommonProxyErrors.db_not_connected_error.value},
+        )
+
+    if user_api_key_dict.user_role != "proxy_admin":
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": "{}, your role={}".format(
+                    CommonProxyErrors.not_allowed_access.value,
+                    user_api_key_dict.user_role,
+                )
+            },
+        )
+
+    response = await prisma_client.db.litellm_invitationlink.delete(
+        where={"id": data.invitation_id}
+    )
+
+    if response is None:
+        raise HTTPException(
+            status_code=400,
+            detail={"error": "Invitation id does not exist in the database."},
+        )
+    return response
+
+
 #### CONFIG MANAGEMENT ####
 @router.post(
     "/config/update",
@@ -11050,6 +11498,7 @@ async def update_config_general_settings(
     "/config/field/info",
     tags=["config.yaml"],
     dependencies=[Depends(user_api_key_auth)],
+    response_model=ConfigFieldInfo,
 )
 async def get_config_general_settings(
     field_name: str,
@@ -11096,10 +11545,9 @@ async def get_config_general_settings(
         general_settings = dict(db_general_settings.param_value)
 
         if field_name in general_settings:
-            return {
-                "field_name": field_name,
-                "field_value": general_settings[field_name],
-            }
+            return ConfigFieldInfo(
+                field_name=field_name, field_value=general_settings[field_name]
+            )
         else:
             raise HTTPException(
                 status_code=400,

litellm/proxy/schema.prisma

@@ -102,6 +102,7 @@ model LiteLLM_UserTable {
     user_alias String? 
     team_id    String?
     organization_id String?
+    password  String?
     teams    String[] @default([])
     user_role  String?
 		max_budget Float?
@@ -117,6 +118,9 @@ model LiteLLM_UserTable {
     model_spend      Json @default("{}")
     model_max_budget Json @default("{}")
     litellm_organization_table LiteLLM_OrganizationTable?   @relation(fields: [organization_id], references: [organization_id])
+    invitations_created LiteLLM_InvitationLink[] @relation("CreatedBy")
+    invitations_updated LiteLLM_InvitationLink[] @relation("UpdatedBy")
+    invitations_user    LiteLLM_InvitationLink[] @relation("UserId")
 }
 
 // Generate Tokens for Proxy
@@ -221,4 +225,22 @@ model LiteLLM_TeamMembership {
   budget_id String?
   litellm_budget_table LiteLLM_BudgetTable?   @relation(fields: [budget_id], references: [budget_id])
   @@id([user_id, team_id])
+}
+
+model LiteLLM_InvitationLink {
+  // use this table to track invite links sent by admin for people to join the proxy
+  id String   @id @default(uuid())
+  user_id String
+  is_accepted Boolean @default(false)
+  accepted_at DateTime? // when link is claimed (user successfully onboards via link)
+  expires_at  DateTime // till when is link valid
+  created_at DateTime // when did admin create the link
+  created_by String // who created the link
+  updated_at DateTime // when was invite status updated
+  updated_by String // who updated the status (admin/user who accepted invite)
+
+  // Relations
+  liteLLM_user_table_user    LiteLLM_UserTable  @relation("UserId", fields: [user_id], references: [user_id])
+  liteLLM_user_table_created LiteLLM_UserTable  @relation("CreatedBy", fields: [created_by], references: [user_id])
+  liteLLM_user_table_updated LiteLLM_UserTable  @relation("UpdatedBy", fields: [updated_by], references: [user_id])
 }

litellm/proxy/utils.py

@@ -35,7 +35,6 @@ from litellm import (
 )
 from litellm.utils import ModelResponseIterator
 from litellm.proxy.hooks.max_budget_limiter import _PROXY_MaxBudgetLimiter
-from litellm.proxy.hooks.tpm_rpm_limiter import _PROXY_MaxTPMRPMLimiter
 from litellm.proxy.hooks.cache_control_check import _PROXY_CacheControlCheck
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.proxy.db.base_client import CustomDB
@@ -81,9 +80,6 @@ class ProxyLogging:
         self.call_details["user_api_key_cache"] = user_api_key_cache
         self.internal_usage_cache = DualCache()
         self.max_parallel_request_limiter = _PROXY_MaxParallelRequestsHandler()
-        self.max_tpm_rpm_limiter = _PROXY_MaxTPMRPMLimiter(
-            internal_cache=self.internal_usage_cache
-        )
         self.max_budget_limiter = _PROXY_MaxBudgetLimiter()
         self.cache_control_check = _PROXY_CacheControlCheck()
         self.alerting: Optional[List] = None
@@ -144,7 +140,6 @@ class ProxyLogging:
         print_verbose(f"INITIALIZING LITELLM CALLBACKS!")
         self.service_logging_obj = ServiceLogging()
         litellm.callbacks.append(self.max_parallel_request_limiter)
-        litellm.callbacks.append(self.max_tpm_rpm_limiter)
         litellm.callbacks.append(self.max_budget_limiter)
         litellm.callbacks.append(self.cache_control_check)
         litellm.callbacks.append(self.service_logging_obj)
@@ -1855,20 +1850,13 @@ async def send_email(receiver_email, subject, html):
     from litellm.proxy.proxy_server import premium_user
     from litellm.proxy.proxy_server import CommonProxyErrors
 
-    # Check if user is premium - This is an Enterprise only Feature
-    if premium_user != True:
-        raise Exception(
-            f"Trying to use Email Alerting\n {CommonProxyErrors.not_premium_user.value}"
-        )
-    # Done Checking
-
     smtp_host = os.getenv("SMTP_HOST")
-    smtp_port = os.getenv("SMTP_PORT", 587)  # default to port 587
+    smtp_port = int(os.getenv("SMTP_PORT", "587"))  # default to port 587
     smtp_username = os.getenv("SMTP_USERNAME")
     smtp_password = os.getenv("SMTP_PASSWORD")
     sender_email = os.getenv("SMTP_SENDER_EMAIL", None)
     if sender_email is None:
-        raise Exception("Trying to use SMTP, but SMTP_SENDER_EMAIL is not set")
+        raise ValueError("Trying to use SMTP, but SMTP_SENDER_EMAIL is not set")
 
     ## EMAIL SETUP ##
     email_message = MIMEMultipart()

litellm/router.py

@@ -38,6 +38,7 @@ from litellm.utils import (
 import copy
 from litellm._logging import verbose_router_logger
 import logging
+from litellm.types.utils import ModelInfo as ModelMapInfo
 from litellm.types.router import (
     Deployment,
     ModelInfo,
@@ -349,17 +350,13 @@ class Router:
     def validate_fallbacks(self, fallback_param: Optional[List]):
         if fallback_param is None:
             return
-        if len(fallback_param) > 0:  # if set
-            ## for dictionary in list, check if only 1 key in dict
-            for _dict in fallback_param:
-                assert isinstance(_dict, dict), "Item={}, not a dictionary".format(
-                    _dict
-                )
-                assert (
-                    len(_dict.keys()) == 1
-                ), "Only 1 key allows in dictionary. You set={} for dict={}".format(
-                    len(_dict.keys()), _dict
-                )
+
+        for fallback_dict in fallback_param:
+            if not isinstance(fallback_dict, dict):
+                raise ValueError(f"Item '{fallback_dict}' is not a dictionary.")
+            if len(fallback_dict) != 1:
+                raise ValueError(
+                    f"Dictionary '{fallback_dict}' must have exactly one key, but has {len(fallback_dict)} keys.")
 
     def routing_strategy_init(self, routing_strategy: str, routing_strategy_args: dict):
         if routing_strategy == "least-busy":
@@ -3065,16 +3062,31 @@ class Router:
                 try:
                     model_info = litellm.get_model_info(model=litellm_params.model)
                 except Exception as e:
-                    continue
+                    model_info = None
                 # get llm provider
                 try:
                     model, llm_provider, _, _ = litellm.get_llm_provider(
                         model=litellm_params.model,
                         custom_llm_provider=litellm_params.custom_llm_provider,
                     )
-                except Exception as e:
+                except litellm.exceptions.BadRequestError as e:
                     continue
 
+                if model_info is None:
+                    supported_openai_params = litellm.get_supported_openai_params(
+                        model=model, custom_llm_provider=llm_provider
+                    )
+                    model_info = ModelMapInfo(
+                        max_tokens=None,
+                        max_input_tokens=None,
+                        max_output_tokens=None,
+                        input_cost_per_token=0,
+                        output_cost_per_token=0,
+                        litellm_provider=llm_provider,
+                        mode="chat",
+                        supported_openai_params=supported_openai_params,
+                    )
+
                 if model_group_info is None:
                     model_group_info = ModelGroupInfo(
                         model_group=model_group, providers=[llm_provider], **model_info  # type: ignore
@@ -3089,18 +3101,26 @@ class Router:
                     # supports_function_calling == True
                     if llm_provider not in model_group_info.providers:
                         model_group_info.providers.append(llm_provider)
-                    if model_info.get("max_input_tokens", None) is not None and (
-                        model_group_info.max_input_tokens is None
-                        or model_info["max_input_tokens"]
-                        > model_group_info.max_input_tokens
+                    if (
+                        model_info.get("max_input_tokens", None) is not None
+                        and model_info["max_input_tokens"] is not None
+                        and (
+                            model_group_info.max_input_tokens is None
+                            or model_info["max_input_tokens"]
+                            > model_group_info.max_input_tokens
+                        )
                     ):
                         model_group_info.max_input_tokens = model_info[
                             "max_input_tokens"
                         ]
-                    if model_info.get("max_output_tokens", None) is not None and (
-                        model_group_info.max_output_tokens is None
-                        or model_info["max_output_tokens"]
-                        > model_group_info.max_output_tokens
+                    if (
+                        model_info.get("max_output_tokens", None) is not None
+                        and model_info["max_output_tokens"] is not None
+                        and (
+                            model_group_info.max_output_tokens is None
+                            or model_info["max_output_tokens"]
+                            > model_group_info.max_output_tokens
+                        )
                     ):
                         model_group_info.max_output_tokens = model_info[
                             "max_output_tokens"
@@ -3124,19 +3144,26 @@ class Router:
                     if (
                         model_info.get("supports_parallel_function_calling", None)
                         is not None
-                        and model_info["supports_parallel_function_calling"] == True  # type: ignore
+                        and model_info["supports_parallel_function_calling"] is True  # type: ignore
                     ):
                         model_group_info.supports_parallel_function_calling = True
                     if (
                         model_info.get("supports_vision", None) is not None
-                        and model_info["supports_vision"] == True  # type: ignore
+                        and model_info["supports_vision"] is True  # type: ignore
                     ):
                         model_group_info.supports_vision = True
                     if (
                         model_info.get("supports_function_calling", None) is not None
-                        and model_info["supports_function_calling"] == True  # type: ignore
+                        and model_info["supports_function_calling"] is True  # type: ignore
                     ):
                         model_group_info.supports_function_calling = True
+                    if (
+                        model_info.get("supported_openai_params", None) is not None
+                        and model_info["supported_openai_params"] is not None
+                    ):
+                        model_group_info.supported_openai_params = model_info[
+                            "supported_openai_params"
+                        ]
 
         return model_group_info
 

litellm/tests/test_bedrock_completion.py

@@ -13,7 +13,7 @@ import pytest
 import litellm
 from litellm import embedding, completion, completion_cost, Timeout, ModelResponse
 from litellm import RateLimitError
-from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.llms.custom_httpx.http_handler import HTTPHandler, AsyncHTTPHandler
 from unittest.mock import patch, AsyncMock, Mock
 
 # litellm.num_retries = 3
@@ -239,25 +239,6 @@ def test_completion_bedrock_claude_sts_oidc_auth():
         pytest.fail(f"Error occurred: {e}")
 
 
-def test_bedrock_extra_headers():
-    try:
-        litellm.set_verbose = True
-        response: ModelResponse = completion(
-            model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0",
-            messages=messages,
-            max_tokens=10,
-            temperature=0.78,
-            extra_headers={"x-key": "x_key_value"},
-        )
-        # Add any assertions here to check the response
-        assert len(response.choices) > 0
-        assert len(response.choices[0].message.content) > 0
-    except RateLimitError:
-        pass
-    except Exception as e:
-        pytest.fail(f"Error occurred: {e}")
-
-
 def test_bedrock_claude_3():
     try:
         litellm.set_verbose = True
@@ -536,3 +517,32 @@ def test_bedrock_ptu():
             == "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A888602223428%3Aprovisioned-model%2F8fxff74qyhs3/invoke"
         )
         mock_client_post.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_bedrock_extra_headers():
+    """
+    Check if a url with 'modelId' passed in, is created correctly
+
+    Reference: https://github.com/BerriAI/litellm/issues/3805
+    """
+    client = AsyncHTTPHandler()
+
+    with patch.object(client, "post", new=AsyncMock()) as mock_client_post:
+        litellm.set_verbose = True
+        from openai.types.chat import ChatCompletion
+
+        try:
+            response = await litellm.acompletion(
+                model="anthropic.claude-3-sonnet-20240229-v1:0",
+                messages=[{"role": "user", "content": "What's AWS?"}],
+                client=client,
+                extra_headers={"test": "hello world"},
+            )
+        except Exception as e:
+            pass
+
+        print(f"mock_client_post.call_args: {mock_client_post.call_args}")
+        assert "test" in mock_client_post.call_args.kwargs["headers"]
+        assert mock_client_post.call_args.kwargs["headers"]["test"] == "hello world"
+        mock_client_post.assert_called_once()

litellm/tests/test_max_tpm_rpm_limiter.py

@@ -1,162 +1,163 @@
+### REPLACED BY 'test_parallel_request_limiter.py' ###
 # What is this?
 ## Unit tests for the max tpm / rpm limiter hook for proxy
 
-import sys, os, asyncio, time, random
-from datetime import datetime
-import traceback
-from dotenv import load_dotenv
-from typing import Optional
+# import sys, os, asyncio, time, random
+# from datetime import datetime
+# import traceback
+# from dotenv import load_dotenv
+# from typing import Optional
 
-load_dotenv()
-import os
+# load_dotenv()
+# import os
 
-sys.path.insert(
-    0, os.path.abspath("../..")
-)  # Adds the parent directory to the system path
-import pytest
-import litellm
-from litellm import Router
-from litellm.proxy.utils import ProxyLogging, hash_token
-from litellm.proxy._types import UserAPIKeyAuth
-from litellm.caching import DualCache, RedisCache
-from litellm.proxy.hooks.tpm_rpm_limiter import _PROXY_MaxTPMRPMLimiter
-from datetime import datetime
+# sys.path.insert(
+#     0, os.path.abspath("../..")
+# )  # Adds the parent directory to the system path
+# import pytest
+# import litellm
+# from litellm import Router
+# from litellm.proxy.utils import ProxyLogging, hash_token
+# from litellm.proxy._types import UserAPIKeyAuth
+# from litellm.caching import DualCache, RedisCache
+# from litellm.proxy.hooks.tpm_rpm_limiter import _PROXY_MaxTPMRPMLimiter
+# from datetime import datetime
 
 
-@pytest.mark.asyncio
-async def test_pre_call_hook_rpm_limits():
-    """
-    Test if error raised on hitting rpm limits
-    """
-    litellm.set_verbose = True
-    _api_key = hash_token("sk-12345")
-    user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, tpm_limit=9, rpm_limit=1)
-    local_cache = DualCache()
-    # redis_usage_cache = RedisCache()
+# @pytest.mark.asyncio
+# async def test_pre_call_hook_rpm_limits():
+#     """
+#     Test if error raised on hitting rpm limits
+#     """
+#     litellm.set_verbose = True
+#     _api_key = hash_token("sk-12345")
+#     user_api_key_dict = UserAPIKeyAuth(api_key=_api_key, tpm_limit=9, rpm_limit=1)
+#     local_cache = DualCache()
+#     # redis_usage_cache = RedisCache()
 
-    local_cache.set_cache(
-        key=_api_key, value={"api_key": _api_key, "tpm_limit": 9, "rpm_limit": 1}
-    )
+#     local_cache.set_cache(
+#         key=_api_key, value={"api_key": _api_key, "tpm_limit": 9, "rpm_limit": 1}
+#     )
 
-    tpm_rpm_limiter = _PROXY_MaxTPMRPMLimiter(internal_cache=DualCache())
+#     tpm_rpm_limiter = _PROXY_MaxTPMRPMLimiter(internal_cache=DualCache())
 
-    await tpm_rpm_limiter.async_pre_call_hook(
-        user_api_key_dict=user_api_key_dict, cache=local_cache, data={}, call_type=""
-    )
+#     await tpm_rpm_limiter.async_pre_call_hook(
+#         user_api_key_dict=user_api_key_dict, cache=local_cache, data={}, call_type=""
+#     )
 
-    kwargs = {"litellm_params": {"metadata": {"user_api_key": _api_key}}}
+#     kwargs = {"litellm_params": {"metadata": {"user_api_key": _api_key}}}
 
-    await tpm_rpm_limiter.async_log_success_event(
-        kwargs=kwargs,
-        response_obj="",
-        start_time="",
-        end_time="",
-    )
+#     await tpm_rpm_limiter.async_log_success_event(
+#         kwargs=kwargs,
+#         response_obj="",
+#         start_time="",
+#         end_time="",
+#     )
 
-    ## Expected cache val: {"current_requests": 0, "current_tpm": 0, "current_rpm": 1}
+#     ## Expected cache val: {"current_requests": 0, "current_tpm": 0, "current_rpm": 1}
 
-    try:
-        await tpm_rpm_limiter.async_pre_call_hook(
-            user_api_key_dict=user_api_key_dict,
-            cache=local_cache,
-            data={},
-            call_type="",
-        )
+#     try:
+#         await tpm_rpm_limiter.async_pre_call_hook(
+#             user_api_key_dict=user_api_key_dict,
+#             cache=local_cache,
+#             data={},
+#             call_type="",
+#         )
 
-        pytest.fail(f"Expected call to fail")
-    except Exception as e:
-        assert e.status_code == 429
+#         pytest.fail(f"Expected call to fail")
+#     except Exception as e:
+#         assert e.status_code == 429
 
 
-@pytest.mark.asyncio
-async def test_pre_call_hook_team_rpm_limits(
-    _redis_usage_cache: Optional[RedisCache] = None,
-):
-    """
-    Test if error raised on hitting team rpm limits
-    """
-    litellm.set_verbose = True
-    _api_key = "sk-12345"
-    _team_id = "unique-team-id"
-    _user_api_key_dict = {
-        "api_key": _api_key,
-        "max_parallel_requests": 1,
-        "tpm_limit": 9,
-        "rpm_limit": 10,
-        "team_rpm_limit": 1,
-        "team_id": _team_id,
-    }
-    user_api_key_dict = UserAPIKeyAuth(**_user_api_key_dict)  # type: ignore
-    _api_key = hash_token(_api_key)
-    local_cache = DualCache()
-    local_cache.set_cache(key=_api_key, value=_user_api_key_dict)
-    internal_cache = DualCache(redis_cache=_redis_usage_cache)
-    tpm_rpm_limiter = _PROXY_MaxTPMRPMLimiter(internal_cache=internal_cache)
-    await tpm_rpm_limiter.async_pre_call_hook(
-        user_api_key_dict=user_api_key_dict, cache=local_cache, data={}, call_type=""
-    )
+# @pytest.mark.asyncio
+# async def test_pre_call_hook_team_rpm_limits(
+#     _redis_usage_cache: Optional[RedisCache] = None,
+# ):
+#     """
+#     Test if error raised on hitting team rpm limits
+#     """
+#     litellm.set_verbose = True
+#     _api_key = "sk-12345"
+#     _team_id = "unique-team-id"
+#     _user_api_key_dict = {
+#         "api_key": _api_key,
+#         "max_parallel_requests": 1,
+#         "tpm_limit": 9,
+#         "rpm_limit": 10,
+#         "team_rpm_limit": 1,
+#         "team_id": _team_id,
+#     }
+#     user_api_key_dict = UserAPIKeyAuth(**_user_api_key_dict)  # type: ignore
+#     _api_key = hash_token(_api_key)
+#     local_cache = DualCache()
+#     local_cache.set_cache(key=_api_key, value=_user_api_key_dict)
+#     internal_cache = DualCache(redis_cache=_redis_usage_cache)
+#     tpm_rpm_limiter = _PROXY_MaxTPMRPMLimiter(internal_cache=internal_cache)
+#     await tpm_rpm_limiter.async_pre_call_hook(
+#         user_api_key_dict=user_api_key_dict, cache=local_cache, data={}, call_type=""
+#     )
 
-    kwargs = {
-        "litellm_params": {
-            "metadata": {"user_api_key": _api_key, "user_api_key_team_id": _team_id}
-        }
-    }
+#     kwargs = {
+#         "litellm_params": {
+#             "metadata": {"user_api_key": _api_key, "user_api_key_team_id": _team_id}
+#         }
+#     }
 
-    await tpm_rpm_limiter.async_log_success_event(
-        kwargs=kwargs,
-        response_obj="",
-        start_time="",
-        end_time="",
-    )
+#     await tpm_rpm_limiter.async_log_success_event(
+#         kwargs=kwargs,
+#         response_obj="",
+#         start_time="",
+#         end_time="",
+#     )
 
-    print(f"local_cache: {local_cache}")
+#     print(f"local_cache: {local_cache}")
 
-    ## Expected cache val: {"current_requests": 0, "current_tpm": 0, "current_rpm": 1}
+#     ## Expected cache val: {"current_requests": 0, "current_tpm": 0, "current_rpm": 1}
 
-    try:
-        await tpm_rpm_limiter.async_pre_call_hook(
-            user_api_key_dict=user_api_key_dict,
-            cache=local_cache,
-            data={},
-            call_type="",
-        )
+#     try:
+#         await tpm_rpm_limiter.async_pre_call_hook(
+#             user_api_key_dict=user_api_key_dict,
+#             cache=local_cache,
+#             data={},
+#             call_type="",
+#         )
 
-        pytest.fail(f"Expected call to fail")
-    except Exception as e:
-        assert e.status_code == 429  # type: ignore
+#         pytest.fail(f"Expected call to fail")
+#     except Exception as e:
+#         assert e.status_code == 429  # type: ignore
 
 
-@pytest.mark.asyncio
-async def test_namespace():
-    """
-    - test if default namespace set via `proxyconfig._init_cache`
-    - respected for tpm/rpm caching
-    """
-    from litellm.proxy.proxy_server import ProxyConfig
+# @pytest.mark.asyncio
+# async def test_namespace():
+#     """
+#     - test if default namespace set via `proxyconfig._init_cache`
+#     - respected for tpm/rpm caching
+#     """
+#     from litellm.proxy.proxy_server import ProxyConfig
 
-    redis_usage_cache: Optional[RedisCache] = None
-    cache_params = {"type": "redis", "namespace": "litellm_default"}
+#     redis_usage_cache: Optional[RedisCache] = None
+#     cache_params = {"type": "redis", "namespace": "litellm_default"}
 
-    ## INIT CACHE ##
-    proxy_config = ProxyConfig()
-    setattr(litellm.proxy.proxy_server, "proxy_config", proxy_config)
+#     ## INIT CACHE ##
+#     proxy_config = ProxyConfig()
+#     setattr(litellm.proxy.proxy_server, "proxy_config", proxy_config)
 
-    proxy_config._init_cache(cache_params=cache_params)
+#     proxy_config._init_cache(cache_params=cache_params)
 
-    redis_cache: Optional[RedisCache] = getattr(
-        litellm.proxy.proxy_server, "redis_usage_cache"
-    )
+#     redis_cache: Optional[RedisCache] = getattr(
+#         litellm.proxy.proxy_server, "redis_usage_cache"
+#     )
 
-    ## CHECK IF NAMESPACE SET ##
-    assert redis_cache.namespace == "litellm_default"
+#     ## CHECK IF NAMESPACE SET ##
+#     assert redis_cache.namespace == "litellm_default"
 
-    ## CHECK IF TPM/RPM RATE LIMITING WORKS ##
-    await test_pre_call_hook_team_rpm_limits(_redis_usage_cache=redis_cache)
-    current_date = datetime.now().strftime("%Y-%m-%d")
-    current_hour = datetime.now().strftime("%H")
-    current_minute = datetime.now().strftime("%M")
-    precise_minute = f"{current_date}-{current_hour}-{current_minute}"
+#     ## CHECK IF TPM/RPM RATE LIMITING WORKS ##
+#     await test_pre_call_hook_team_rpm_limits(_redis_usage_cache=redis_cache)
+#     current_date = datetime.now().strftime("%Y-%m-%d")
+#     current_hour = datetime.now().strftime("%H")
+#     current_minute = datetime.now().strftime("%M")
+#     precise_minute = f"{current_date}-{current_hour}-{current_minute}"
 
-    cache_key = "litellm_default:usage:{}".format(precise_minute)
-    value = await redis_cache.async_get_cache(key=cache_key)
-    assert value is not None
+#     cache_key = "litellm_default:usage:{}".format(precise_minute)
+#     value = await redis_cache.async_get_cache(key=cache_key)
+#     assert value is not None

litellm/tests/test_parallel_request_limiter.py

@@ -229,17 +229,21 @@ async def test_pre_call_hook_user_tpm_limits():
     """
     Test if error raised on hitting tpm limits
     """
+    local_cache = DualCache()
     # create user with tpm/rpm limits
+    user_id = "test-user"
+    user_obj = {"tpm_limit": 9, "rpm_limit": 10}
+
+    local_cache.set_cache(key=user_id, value=user_obj)
 
     _api_key = "sk-12345"
     user_api_key_dict = UserAPIKeyAuth(
         api_key=_api_key,
-        user_id="ishaan",
-        user_id_rate_limits={"tpm_limit": 9, "rpm_limit": 10},
+        user_id=user_id,
     )
     res = dict(user_api_key_dict)
     print("dict user", res)
-    local_cache = DualCache()
+
     parallel_request_handler = MaxParallelRequestsHandler()
 
     await parallel_request_handler.async_pre_call_hook(
@@ -248,7 +252,7 @@ async def test_pre_call_hook_user_tpm_limits():
 
     kwargs = {
         "litellm_params": {
-            "metadata": {"user_api_key_user_id": "ishaan", "user_api_key": "gm"}
+            "metadata": {"user_api_key_user_id": user_id, "user_api_key": "gm"}
         }
     }
 
@@ -734,7 +738,7 @@ async def test_bad_router_call():
     request_count_api_key = f"{_api_key}::{precise_minute}::request_count"
 
     assert (
-        parallel_request_handler.user_api_key_cache.get_cache(
+        parallel_request_handler.user_api_key_cache.get_cache(  # type: ignore
             key=request_count_api_key
         )["current_requests"]
         == 1
@@ -751,7 +755,7 @@ async def test_bad_router_call():
     except:
         pass
     assert (
-        parallel_request_handler.user_api_key_cache.get_cache(
+        parallel_request_handler.user_api_key_cache.get_cache(  # type: ignore
             key=request_count_api_key
         )["current_requests"]
         == 0

litellm/types/router.py

@@ -1,9 +1,15 @@
+"""
+    litellm.Router Types - includes RouterConfig, UpdateRouterConfig, ModelInfo etc
+"""
+
 from typing import List, Optional, Union, Dict, Tuple, Literal, TypedDict
+import uuid
+import enum
 import httpx
-from pydantic import BaseModel, validator, Field
+from pydantic import BaseModel, Field
+import datetime
 from .completion import CompletionRequest
 from .embedding import EmbeddingRequest
-import uuid, enum
 
 
 class ModelConfig(BaseModel):
@@ -76,6 +82,12 @@ class ModelInfo(BaseModel):
     db_model: bool = (
         False  # used for proxy - to separate models which are stored in the db vs. config.
     )
+    updated_at: Optional[datetime.datetime] = None
+    updated_by: Optional[str] = None
+
+    created_at: Optional[datetime.datetime] = None
+    created_by: Optional[str] = None
+
     base_model: Optional[str] = (
         None  # specify if the base model is azure/gpt-3.5-turbo etc for accurate cost tracking
     )
@@ -426,3 +438,4 @@ class ModelGroupInfo(BaseModel):
     supports_parallel_function_calling: bool = Field(default=False)
     supports_vision: bool = Field(default=False)
     supports_function_calling: bool = Field(default=False)
+    supported_openai_params: List[str] = Field(default=[])

litellm/types/utils.py

@@ -15,10 +15,17 @@ class ProviderField(TypedDict):
 
 
 class ModelInfo(TypedDict):
-    max_tokens: int
-    max_input_tokens: int
-    max_output_tokens: int
+    """
+    Model info for a given model, this is information found in litellm.model_prices_and_context_window.json
+    """
+
+    max_tokens: Optional[int]
+    max_input_tokens: Optional[int]
+    max_output_tokens: Optional[int]
     input_cost_per_token: float
     output_cost_per_token: float
     litellm_provider: str
-    mode: str
+    mode: Literal[
+        "completion", "embedding", "image_generation", "chat", "audio_transcription"
+    ]
+    supported_openai_params: Optional[List[str]]

litellm/utils.py

@@ -5797,30 +5797,11 @@ def get_optional_params(
             model=model, custom_llm_provider=custom_llm_provider
         )
         _check_valid_arg(supported_params=supported_params)
-        if temperature is not None:
-            if (
-                temperature == 0 and model == "mistralai/Mistral-7B-Instruct-v0.1"
-            ):  # this model does no support temperature == 0
-                temperature = 0.0001  # close to 0
-            optional_params["temperature"] = temperature
-        if top_p:
-            optional_params["top_p"] = top_p
-        if n:
-            optional_params["n"] = n
-        if stream:
-            optional_params["stream"] = stream
-        if stop:
-            optional_params["stop"] = stop
-        if max_tokens:
-            optional_params["max_tokens"] = max_tokens
-        if presence_penalty:
-            optional_params["presence_penalty"] = presence_penalty
-        if frequency_penalty:
-            optional_params["frequency_penalty"] = frequency_penalty
-        if logit_bias:
-            optional_params["logit_bias"] = logit_bias
-        if user:
-            optional_params["user"] = user
+        optional_params = litellm.DeepInfraConfig().map_openai_params(
+            non_default_params=non_default_params,
+            optional_params=optional_params,
+            model=model,
+        )
     elif custom_llm_provider == "perplexity":
         supported_params = get_supported_openai_params(
             model=model, custom_llm_provider=custom_llm_provider
@@ -6604,19 +6585,7 @@ def get_supported_openai_params(
     elif custom_llm_provider == "petals":
         return ["max_tokens", "temperature", "top_p", "stream"]
     elif custom_llm_provider == "deepinfra":
-        return [
-            "temperature",
-            "top_p",
-            "n",
-            "stream",
-            "stop",
-            "max_tokens",
-            "presence_penalty",
-            "frequency_penalty",
-            "logit_bias",
-            "user",
-            "response_format",
-        ]
+        return litellm.DeepInfraConfig().get_supported_openai_params()
     elif custom_llm_provider == "perplexity":
         return [
             "temperature",
@@ -7107,6 +7076,7 @@ def get_model_info(model: str) -> ModelInfo:
             - output_cost_per_token (float): The cost per token for output.
             - litellm_provider (str): The provider of the model (e.g., "openai").
             - mode (str): The mode of the model (e.g., "chat" or "completion").
+            - supported_openai_params (List[str]): A list of supported OpenAI parameters for the model.
 
     Raises:
         Exception: If the model is not mapped yet.
@@ -7118,9 +7088,11 @@ def get_model_info(model: str) -> ModelInfo:
             "input_cost_per_token": 0.00003,
             "output_cost_per_token": 0.00006,
             "litellm_provider": "openai",
-            "mode": "chat"
+            "mode": "chat",
+            "supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"]
         }
     """
+    supported_openai_params: Union[List[str], None] = []
 
     def _get_max_position_embeddings(model_name):
         # Construct the URL for the config.json file
@@ -7148,9 +7120,18 @@ def get_model_info(model: str) -> ModelInfo:
         azure_llms = litellm.azure_llms
         if model in azure_llms:
             model = azure_llms[model]
-        if model in litellm.model_cost:
-            return litellm.model_cost[model]
-        model, custom_llm_provider, _, _ = get_llm_provider(model=model)
+        ##########################
+        # Get custom_llm_provider
+        split_model, custom_llm_provider = model, ""
+        try:
+            split_model, custom_llm_provider, _, _ = get_llm_provider(model=model)
+        except:
+            pass
+        #########################
+
+        supported_openai_params = litellm.get_supported_openai_params(
+            model=model, custom_llm_provider=custom_llm_provider
+        )
         if custom_llm_provider == "huggingface":
             max_tokens = _get_max_position_embeddings(model_name=model)
             return {
@@ -7159,15 +7140,26 @@ def get_model_info(model: str) -> ModelInfo:
                 "output_cost_per_token": 0,
                 "litellm_provider": "huggingface",
                 "mode": "chat",
+                "supported_openai_params": supported_openai_params,
             }
         else:
             """
-            Check if model in model cost map
+            Check if:
+            1. 'model' in litellm.model_cost. Checks "groq/llama3-8b-8192" in  litellm.model_cost
+            2. 'split_model' in litellm.model_cost. Checks "llama3-8b-8192" in litellm.model_cost
             """
             if model in litellm.model_cost:
-                return litellm.model_cost[model]
+                _model_info = litellm.model_cost[model]
+                _model_info["supported_openai_params"] = supported_openai_params
+                return _model_info
+            if split_model in litellm.model_cost:
+                _model_info = litellm.model_cost[split_model]
+                _model_info["supported_openai_params"] = supported_openai_params
+                return _model_info
             else:
-                raise Exception()
+                raise ValueError(
+                    "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
+                )
     except:
         raise Exception(
             "This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm"
-version = "1.38.10"
+version = "1.38.11"
 description = "Library to easily interface with LLM API providers"
 authors = ["BerriAI"]
 license = "MIT"
@@ -79,7 +79,7 @@ requires = ["poetry-core", "wheel"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "1.38.10"
+version = "1.38.11"
 version_files = [
     "pyproject.toml:^version"
 ]

schema.prisma

@@ -102,6 +102,7 @@ model LiteLLM_UserTable {
     user_alias String? 
     team_id    String?
     organization_id String?
+    password  String?
     teams    String[] @default([])
     user_role  String?
 		max_budget Float?
@@ -117,6 +118,9 @@ model LiteLLM_UserTable {
     model_spend      Json @default("{}")
     model_max_budget Json @default("{}")
     litellm_organization_table LiteLLM_OrganizationTable?   @relation(fields: [organization_id], references: [organization_id])
+    invitations_created LiteLLM_InvitationLink[] @relation("CreatedBy")
+    invitations_updated LiteLLM_InvitationLink[] @relation("UpdatedBy")
+    invitations_user    LiteLLM_InvitationLink[] @relation("UserId")
 }
 
 // Generate Tokens for Proxy
@@ -222,3 +226,21 @@ model LiteLLM_TeamMembership {
   litellm_budget_table LiteLLM_BudgetTable?   @relation(fields: [budget_id], references: [budget_id])
   @@id([user_id, team_id])
 }
+
+model LiteLLM_InvitationLink {
+  // use this table to track invite links sent by admin for people to join the proxy
+  id String   @id @default(uuid())
+  user_id String
+  is_accepted Boolean @default(false)
+  accepted_at DateTime? // when link is claimed (user successfully onboards via link)
+  expires_at  DateTime // till when is link valid
+  created_at DateTime // when did admin create the link
+  created_by String // who created the link
+  updated_at DateTime // when was invite status updated
+  updated_by String // who updated the status (admin/user who accepted invite)
+
+  // Relations
+  liteLLM_user_table_user    LiteLLM_UserTable  @relation("UserId", fields: [user_id], references: [user_id])
+  liteLLM_user_table_created LiteLLM_UserTable  @relation("CreatedBy", fields: [created_by], references: [user_id])
+  liteLLM_user_table_updated LiteLLM_UserTable  @relation("UpdatedBy", fields: [updated_by], references: [user_id])
+}

ui/litellm-dashboard/out/index.html

@@ -1 +1 @@
-<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-e85084d25f9ae5e4.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-f960ab1e6d32b002.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-04708d7d4a17c1ee.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-e85084d25f9ae5e4.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/5d93d4a9fa59d72f.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[39712,[\"936\",\"static/chunks/2f6dbc85-052c4579f80d66ae.js\",\"608\",\"static/chunks/608-d128caa3cfe973c1.js\",\"931\",\"static/chunks/app/page-e266cb0126026d40.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/5d93d4a9fa59d72f.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"dYIEEO-62OCgyckEhgBd-\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>
+<!DOCTYPE html><html id="__next_error__"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" as="script" fetchPriority="low" href="/ui/_next/static/chunks/webpack-e85084d25f9ae5e4.js" crossorigin=""/><script src="/ui/_next/static/chunks/fd9d1056-f960ab1e6d32b002.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/69-04708d7d4a17c1ee.js" async="" crossorigin=""></script><script src="/ui/_next/static/chunks/main-app-9b4fb13a7db53edf.js" async="" crossorigin=""></script><title>LiteLLM Dashboard</title><meta name="description" content="LiteLLM Proxy Admin UI"/><link rel="icon" href="/ui/favicon.ico" type="image/x-icon" sizes="16x16"/><meta name="next-size-adjust"/><script src="/ui/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" crossorigin="" noModule=""></script></head><body><script src="/ui/_next/static/chunks/webpack-e85084d25f9ae5e4.js" crossorigin="" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])</script><script>self.__next_f.push([1,"1:HL[\"/ui/_next/static/media/c9a5bc6a7c948fb0-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/ui/_next/static/css/5d93d4a9fa59d72f.css\",\"style\",{\"crossOrigin\":\"\"}]\n0:\"$L3\"\n"])</script><script>self.__next_f.push([1,"4:I[47690,[],\"\"]\n6:I[77831,[],\"\"]\n7:I[39712,[\"936\",\"static/chunks/2f6dbc85-052c4579f80d66ae.js\",\"608\",\"static/chunks/608-d128caa3cfe973c1.js\",\"931\",\"static/chunks/app/page-76d278f96a0e9768.js\"],\"\"]\n8:I[5613,[],\"\"]\n9:I[31778,[],\"\"]\nb:I[48955,[],\"\"]\nc:[]\n"])</script><script>self.__next_f.push([1,"3:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/ui/_next/static/css/5d93d4a9fa59d72f.css\",\"precedence\":\"next\",\"crossOrigin\":\"\"}]],[\"$\",\"$L4\",null,{\"buildId\":\"D_ZUmMtLMPSa4aQQUJtKt\",\"assetPrefix\":\"/ui\",\"initialCanonicalUrl\":\"/\",\"initialTree\":[\"\",{\"children\":[\"__PAGE__\",{}]},\"$undefined\",\"$undefined\",true],\"initialSeedData\":[\"\",{\"children\":[\"__PAGE__\",{},[\"$L5\",[\"$\",\"$L6\",null,{\"propsForComponent\":{\"params\":{}},\"Component\":\"$7\",\"isStaticGeneration\":true}],null]]},[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"children\":[\"$\",\"body\",null,{\"className\":\"__className_c23dc8\",\"children\":[\"$\",\"$L8\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"loadingScripts\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"errorScripts\":\"$undefined\",\"template\":[\"$\",\"$L9\",null,{}],\"templateStyles\":\"$undefined\",\"templateScripts\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"styles\":null}]}]}],null]],\"initialHead\":[false,\"$La\"],\"globalErrorComponent\":\"$b\",\"missingSlots\":\"$Wc\"}]]\n"])</script><script>self.__next_f.push([1,"a:[[\"$\",\"meta\",\"0\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"1\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"2\",{\"children\":\"LiteLLM Dashboard\"}],[\"$\",\"meta\",\"3\",{\"name\":\"description\",\"content\":\"LiteLLM Proxy Admin UI\"}],[\"$\",\"link\",\"4\",{\"rel\":\"icon\",\"href\":\"/ui/favicon.ico\",\"type\":\"image/x-icon\",\"sizes\":\"16x16\"}],[\"$\",\"meta\",\"5\",{\"name\":\"next-size-adjust\"}]]\n5:null\n"])</script><script>self.__next_f.push([1,""])</script></body></html>

ui/litellm-dashboard/out/index.txt

@@ -1,7 +1,7 @@
 2:I[77831,[],""]
-3:I[39712,["936","static/chunks/2f6dbc85-052c4579f80d66ae.js","608","static/chunks/608-d128caa3cfe973c1.js","931","static/chunks/app/page-e266cb0126026d40.js"],""]
+3:I[39712,["936","static/chunks/2f6dbc85-052c4579f80d66ae.js","608","static/chunks/608-d128caa3cfe973c1.js","931","static/chunks/app/page-76d278f96a0e9768.js"],""]
 4:I[5613,[],""]
 5:I[31778,[],""]
-0:["dYIEEO-62OCgyckEhgBd-",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/5d93d4a9fa59d72f.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
+0:["D_ZUmMtLMPSa4aQQUJtKt",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},["$L1",["$","$L2",null,{"propsForComponent":{"params":{}},"Component":"$3","isStaticGeneration":true}],null]]},[null,["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_c23dc8","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"loading":"$undefined","loadingStyles":"$undefined","loadingScripts":"$undefined","hasLoading":false,"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[],"styles":null}]}]}],null]],[[["$","link","0",{"rel":"stylesheet","href":"/ui/_next/static/css/5d93d4a9fa59d72f.css","precedence":"next","crossOrigin":""}]],"$L6"]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/ui/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","meta","5",{"name":"next-size-adjust"}]]
 1:null

ui/litellm-dashboard/src/app/model_hub/page.tsx

@@ -0,0 +1,25 @@
+"use client";
+import React, { Suspense, useEffect, useState } from "react";
+import { useSearchParams } from "next/navigation";
+import { modelHubCall } from "@/components/networking";
+import ModelHub from "@/components/model_hub";
+
+export default function PublicModelHub() {
+  const searchParams = useSearchParams();
+  const key = searchParams.get("key");
+  const [accessToken, setAccessToken] = useState<string | null>(null);
+
+  useEffect(() => {
+    if (!key) {
+      return;
+    }
+    setAccessToken(key);
+  }, [key]);
+  /**
+   * populate navbar
+   *
+   */
+  return (
+    <ModelHub accessToken={accessToken} publicPage={true} premiumUser={false} />
+  );
+}

ui/litellm-dashboard/src/app/page.tsx

@@ -18,6 +18,30 @@ import Usage from "../components/usage";
 import { jwtDecode } from "jwt-decode";
 import { Typography } from "antd";
 
+export function formatUserRole(userRole: string) {
+  if (!userRole) {
+    return "Undefined Role";
+  }
+  console.log(`Received user role: ${userRole.toLowerCase()}`);
+  console.log(`Received user role length: ${userRole.toLowerCase().length}`);
+  switch (userRole.toLowerCase()) {
+    case "app_owner":
+      return "App Owner";
+    case "demo_app_owner":
+      return "App Owner";
+    case "app_admin":
+      return "Admin";
+    case "proxy_admin":
+      return "Admin";
+    case "proxy_admin_viewer":
+      return "Admin Viewer";
+    case "app_user":
+      return "App User";
+    default:
+      return "Unknown Role";
+  }
+}
+
 const CreateKeyPage = () => {
   const { Title, Paragraph } = Typography;
   const [userRole, setUserRole] = useState("");
@@ -78,30 +102,6 @@ const CreateKeyPage = () => {
     }
   }, [token]);
 
-  function formatUserRole(userRole: string) {
-    if (!userRole) {
-      return "Undefined Role";
-    }
-    console.log(`Received user role: ${userRole.toLowerCase()}`);
-    console.log(`Received user role length: ${userRole.toLowerCase().length}`);
-    switch (userRole.toLowerCase()) {
-      case "app_owner":
-        return "App Owner";
-      case "demo_app_owner":
-        return "App Owner";
-      case "app_admin":
-        return "Admin";
-      case "proxy_admin":
-        return "Admin";
-      case "proxy_admin_viewer":
-        return "Admin Viewer";
-      case "app_user":
-        return "App User";
-      default:
-        return "Unknown Role";
-    }
-  }
-
   return (
     <Suspense fallback={<div>Loading...</div>}>
       <div className="flex flex-col min-h-screen">
@@ -194,17 +194,13 @@ const CreateKeyPage = () => {
               accessToken={accessToken}
               modelData={modelData}
             />
-            ) : page == "model-hub" ? (
-              <ModelHub
-              userID={userID}
-              userRole={userRole}
-              token={token}
+          ) : page == "model-hub" ? (
+            <ModelHub
               accessToken={accessToken}
-              keys={keys}
+              publicPage={false}
               premiumUser={premiumUser}
             />
-            )
-            : (
+          ) : (
             <Usage
               userID={userID}
               userRole={userRole}

ui/litellm-dashboard/src/components/model_dashboard.tsx

@@ -86,6 +86,8 @@ import type { UploadProps } from "antd";
 import { Upload } from "antd";
 import TimeToFirstToken from "./model_metrics/time_to_first_token";
 import DynamicFields from "./model_add/dynamic_form";
+import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
+
 interface ModelDashboardProps {
   accessToken: string | null;
   token: string | null;
@@ -269,6 +271,8 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
   const [selectedProvider, setSelectedProvider] = useState<String>("OpenAI");
   const [healthCheckResponse, setHealthCheckResponse] = useState<string>("");
   const [editModalVisible, setEditModalVisible] = useState<boolean>(false);
+  const [infoModalVisible, setInfoModalVisible] = useState<boolean>(false);
+
   const [selectedModel, setSelectedModel] = useState<any>(null);
   const [availableModelGroups, setAvailableModelGroups] = useState<
     Array<string>
@@ -297,6 +301,15 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
     useState<RetryPolicyObject | null>(null);
   const [defaultRetry, setDefaultRetry] = useState<number>(0);
 
+  function formatCreatedAt(createdAt: string | null) {
+    if (createdAt) {
+      const date = new Date(createdAt);
+      const options = { month: "long", day: "numeric", year: "numeric" };
+      return date.toLocaleDateString("en-US");
+    }
+    return null;
+  }
+
   const EditModelModal: React.FC<EditModelModalProps> = ({
     visible,
     onCancel,
@@ -423,11 +436,21 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
     setEditModalVisible(true);
   };
 
+  const handleInfoClick = (model: any) => {
+    setSelectedModel(model);
+    setInfoModalVisible(true);
+  };
+
   const handleEditCancel = () => {
     setEditModalVisible(false);
     setSelectedModel(null);
   };
 
+  const handleInfoCancel = () => {
+    setInfoModalVisible(false);
+    setSelectedModel(null);
+  };
+
   const handleEditSubmit = async (formValues: Record<string, any>) => {
     // Call API to update team with teamId and values
 
@@ -1038,10 +1061,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                 </Select>
               </div>
               <Card>
-                <Table
-                  className="mt-5"
-                  style={{ maxWidth: "1500px", width: "100%" }}
-                >
+                <Table style={{ maxWidth: "1500px", width: "100%" }}>
                   <TableHead>
                     <TableRow>
                       <TableHeaderCell
@@ -1049,6 +1069,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                           maxWidth: "150px",
                           whiteSpace: "normal",
                           wordBreak: "break-word",
+                          fontSize: "11px",
                         }}
                       >
                         Public Model Name
@@ -1058,6 +1079,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                           maxWidth: "100px",
                           whiteSpace: "normal",
                           wordBreak: "break-word",
+                          fontSize: "11px",
                         }}
                       >
                         Provider
@@ -1068,25 +1090,18 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                             maxWidth: "150px",
                             whiteSpace: "normal",
                             wordBreak: "break-word",
+                            fontSize: "11px",
                           }}
                         >
                           API Base
                         </TableHeaderCell>
                       )}
-                      <TableHeaderCell
-                        style={{
-                          maxWidth: "200px",
-                          whiteSpace: "normal",
-                          wordBreak: "break-word",
-                        }}
-                      >
-                        Extra litellm Params
-                      </TableHeaderCell>
                       <TableHeaderCell
                         style={{
                           maxWidth: "85px",
                           whiteSpace: "normal",
                           wordBreak: "break-word",
+                          fontSize: "11px",
                         }}
                       >
                         Input Price{" "}
@@ -1099,6 +1114,7 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                           maxWidth: "85px",
                           whiteSpace: "normal",
                           wordBreak: "break-word",
+                          fontSize: "11px",
                         }}
                       >
                         Output Price{" "}
@@ -1106,24 +1122,60 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                           /1M Tokens ($)
                         </p>
                       </TableHeaderCell>
+
                       <TableHeaderCell
                         style={{
-                          maxWidth: "120px",
+                          maxWidth: "100px",
                           whiteSpace: "normal",
                           wordBreak: "break-word",
+                          fontSize: "11px",
                         }}
                       >
-                        Max Tokens
+                        {premiumUser ? (
+                          "Created At"
+                        ) : (
+                          <a
+                            href="https://forms.gle/W3U4PZpJGFHWtHyA9"
+                            target="_blank"
+                            style={{ color: "#72bcd4" }}
+                          >
+                            {" "}
+                            ✨ Created At
+                          </a>
+                        )}
+                      </TableHeaderCell>
+                      <TableHeaderCell
+                        style={{
+                          maxWidth: "100px",
+                          whiteSpace: "normal",
+                          wordBreak: "break-word",
+                          fontSize: "11px",
+                        }}
+                      >
+                        {premiumUser ? (
+                          "Created By"
+                        ) : (
+                          <a
+                            href="https://forms.gle/W3U4PZpJGFHWtHyA9"
+                            target="_blank"
+                            style={{ color: "#72bcd4" }}
+                          >
+                            {" "}
+                            ✨ Created By
+                          </a>
+                        )}
                       </TableHeaderCell>
                       <TableHeaderCell
                         style={{
                           maxWidth: "50px",
                           whiteSpace: "normal",
                           wordBreak: "break-word",
+                          fontSize: "11px",
                         }}
                       >
                         Status
                       </TableHeaderCell>
+                      <TableHeaderCell></TableHeaderCell>
                     </TableRow>
                   </TableHead>
                   <TableBody>
@@ -1137,15 +1189,20 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                           selectedModelGroup === ""
                       )
                       .map((model: any, index: number) => (
-                        <TableRow key={index}>
+                        <TableRow
+                          key={index}
+                          style={{ maxHeight: "1px", minHeight: "1px" }}
+                        >
                           <TableCell
                             style={{
-                              maxWidth: "150px",
+                              maxWidth: "100px",
                               whiteSpace: "normal",
                               wordBreak: "break-word",
                             }}
                           >
-                            <Text>{model.model_name}</Text>
+                            <p style={{ fontSize: "10px" }}>
+                              {model.model_name || "-"}
+                            </p>
                           </TableCell>
                           <TableCell
                             style={{
@@ -1154,7 +1211,9 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                               wordBreak: "break-word",
                             }}
                           >
-                            {model.provider}
+                            <p style={{ fontSize: "10px" }}>
+                              {model.provider || "-"}
+                            </p>
                           </TableCell>
                           {userRole === "Admin" && (
                             <TableCell
@@ -1164,30 +1223,46 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                                 wordBreak: "break-word",
                               }}
                             >
-                              {model.api_base}
+                              <Tooltip title={model && model.api_base}>
+                                <pre
+                                  style={{
+                                    maxWidth: "150px",
+                                    whiteSpace: "normal",
+                                    wordBreak: "break-word",
+                                    fontSize: "10px",
+                                  }}
+                                  title={
+                                    model && model.api_base
+                                      ? model.api_base
+                                      : ""
+                                  }
+                                >
+                                  {model && model.api_base
+                                    ? model.api_base.slice(0, 20)
+                                    : "-"}
+                                </pre>
+                              </Tooltip>
                             </TableCell>
                           )}
                           <TableCell
                             style={{
-                              maxWidth: "200px",
+                              maxWidth: "80px",
                               whiteSpace: "normal",
                               wordBreak: "break-word",
                             }}
                           >
-                            <Accordion>
-                              <AccordionHeader>
-                                <Text>Litellm params</Text>
-                              </AccordionHeader>
-                              <AccordionBody>
-                                <pre>
-                                  {JSON.stringify(
-                                    model.cleanedLitellmParams,
-                                    null,
-                                    2
-                                  )}
-                                </pre>
-                              </AccordionBody>
-                            </Accordion>
+                            <pre style={{ fontSize: "10px" }}>
+                              {model.input_cost
+                                ? model.input_cost
+                                : model.litellm_params.input_cost_per_token
+                                  ? (
+                                      Number(
+                                        model.litellm_params
+                                          .input_cost_per_token
+                                      ) * 1000000
+                                    ).toFixed(2)
+                                  : null}
+                            </pre>
                           </TableCell>
                           <TableCell
                             style={{
@@ -1196,31 +1271,33 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                               wordBreak: "break-word",
                             }}
                           >
-                            {model.input_cost ||
-                              model.litellm_params.input_cost_per_token ||
-                              null}
+                            <pre style={{ fontSize: "10px" }}>
+                              {model.output_cost
+                                ? model.output_cost
+                                : model.litellm_params.output_cost_per_token
+                                  ? (
+                                      Number(
+                                        model.litellm_params
+                                          .output_cost_per_token
+                                      ) * 1000000
+                                    ).toFixed(2)
+                                  : null}
+                            </pre>
                           </TableCell>
-                          <TableCell
-                            style={{
-                              maxWidth: "80px",
-                              whiteSpace: "normal",
-                              wordBreak: "break-word",
-                            }}
-                          >
-                            {model.output_cost ||
-                              model.litellm_params.output_cost_per_token ||
-                              null}
-                          </TableCell>
-                          <TableCell
-                            style={{
-                              maxWidth: "120px",
-                              whiteSpace: "normal",
-                              wordBreak: "break-word",
-                            }}
-                          >
+                          <TableCell>
                             <p style={{ fontSize: "10px" }}>
-                              Max Tokens: {model.max_tokens} <br></br>
-                              Max Input Tokens: {model.max_input_tokens}
+                              {premiumUser
+                                ? formatCreatedAt(
+                                    model.model_info.created_at
+                                  ) || "-"
+                                : "-"}
+                            </p>
+                          </TableCell>
+                          <TableCell>
+                            <p style={{ fontSize: "10px" }}>
+                              {premiumUser
+                                ? model.model_info.created_by || "-"
+                                : "-"}
                             </p>
                           </TableCell>
                           <TableCell
@@ -1231,39 +1308,45 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
                             }}
                           >
                             {model.model_info.db_model ? (
-                              <Badge
-                                icon={CheckCircleIcon}
-                                size="xs"
-                                className="text-white"
-                              >
+                              <Badge size="xs" className="text-white">
                                 <p style={{ fontSize: "10px" }}>DB Model</p>
                               </Badge>
                             ) : (
-                              <Badge
-                                icon={XCircleIcon}
-                                size="xs"
-                                className="text-black"
-                              >
+                              <Badge size="xs" className="text-black">
                                 <p style={{ fontSize: "10px" }}>Config Model</p>
                               </Badge>
                             )}
                           </TableCell>
                           <TableCell
                             style={{
-                              maxWidth: "100px",
+                              maxWidth: "150px",
                               whiteSpace: "normal",
                               wordBreak: "break-word",
                             }}
                           >
-                            <Icon
-                              icon={PencilAltIcon}
-                              size="sm"
-                              onClick={() => handleEditClick(model)}
-                            />
-                            <DeleteModelButton
-                              modelID={model.model_info.id}
-                              accessToken={accessToken}
-                            />
+                            <Grid numItems={3}>
+                              <Col>
+                                <Icon
+                                  icon={InformationCircleIcon}
+                                  size="sm"
+                                  onClick={() => handleInfoClick(model)}
+                                />
+                              </Col>
+                              <Col>
+                                <Icon
+                                  icon={PencilAltIcon}
+                                  size="sm"
+                                  onClick={() => handleEditClick(model)}
+                                />
+                              </Col>
+
+                              <Col>
+                                <DeleteModelButton
+                                  modelID={model.model_info.id}
+                                  accessToken={accessToken}
+                                />
+                              </Col>
+                            </Grid>
                           </TableCell>
                         </TableRow>
                       ))}
@@ -1277,6 +1360,18 @@ const ModelDashboard: React.FC<ModelDashboardProps> = ({
               model={selectedModel}
               onSubmit={handleEditSubmit}
             />
+            <Modal
+              title={selectedModel && selectedModel.model_name}
+              visible={infoModalVisible}
+              width={800}
+              footer={null}
+              onCancel={handleInfoCancel}
+            >
+              <Title>Model Info</Title>
+              <SyntaxHighlighter language="json">
+                {selectedModel && JSON.stringify(selectedModel, null, 2)}
+              </SyntaxHighlighter>
+            </Modal>
           </TabPanel>
           <TabPanel className="h-full">
             <Title2 level={2}>Add new model</Title2>

ui/litellm-dashboard/src/components/model_hub.tsx

@@ -1,7 +1,8 @@
 import React, { useEffect, useState } from "react";
+import { useRouter, usePathname, useSearchParams } from "next/navigation";
 
 import { modelHubCall } from "./networking";
-
+import { getConfigFieldSetting, updateConfigFieldSetting } from "./networking";
 import {
   Card,
   Text,
@@ -15,18 +16,14 @@ import {
   TabPanel,
   TabPanels,
 } from "@tremor/react";
-
 import { RightOutlined, CopyOutlined } from "@ant-design/icons";
 
-import { Modal, Tooltip } from "antd";
+import { Modal, Tooltip, message } from "antd";
 import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
 
 interface ModelHubProps {
-  userID: string | null;
-  userRole: string | null;
-  token: string | null;
   accessToken: string | null;
-  keys: any; // Replace with the appropriate type for 'keys' prop
+  publicPage: boolean;
   premiumUser: boolean;
 }
 
@@ -37,46 +34,52 @@ interface ModelInfo {
   supports_vision: boolean;
   max_input_tokens?: number;
   max_output_tokens?: number;
-
-  // Add other properties if needed
+  supported_openai_params?: string[];
 }
 
 const ModelHub: React.FC<ModelHubProps> = ({
-  userID,
-
-  userRole,
-
-  token,
-
   accessToken,
-
-  keys,
-
+  publicPage,
   premiumUser,
 }) => {
+  const [publicPageAllowed, setPublicPageAllowed] = useState<boolean>(false);
   const [modelHubData, setModelHubData] = useState<ModelInfo[] | null>(null);
   const [isModalVisible, setIsModalVisible] = useState(false);
+  const [isPublicPageModalVisible, setIsPublicPageModalVisible] =
+    useState(false);
   const [selectedModel, setSelectedModel] = useState<null | ModelInfo>(null);
+  const router = useRouter();
 
   useEffect(() => {
-    if (!accessToken || !token || !userRole || !userID) {
+    if (!accessToken) {
       return;
     }
 
     const fetchData = async () => {
       try {
-        const _modelHubData = await modelHubCall(accessToken, userID, userRole);
+        const _modelHubData = await modelHubCall(accessToken);
 
         console.log("ModelHubData:", _modelHubData);
 
         setModelHubData(_modelHubData.data);
+
+        getConfigFieldSetting(accessToken, "enable_public_model_hub")
+          .then((data) => {
+            console.log(`data: ${JSON.stringify(data)}`);
+            if (data.field_value == true) {
+              setPublicPageAllowed(true);
+            }
+          })
+          .catch((error) => {
+            // do nothing
+          });
       } catch (error) {
         console.error("There was an error fetching the model data", error);
       }
     };
 
     fetchData();
-  }, [accessToken, token, userRole, userID]);
+  }, [accessToken, publicPage]);
 
   const showModal = (model: ModelInfo) => {
     setSelectedModel(model);
@@ -84,15 +87,29 @@ const ModelHub: React.FC<ModelHubProps> = ({
     setIsModalVisible(true);
   };
 
+  const goToPublicModelPage = () => {
+    router.replace(`/model_hub?key=${accessToken}`);
+  };
+  const handleMakePublicPage = async () => {
+    if (!accessToken) {
+      return;
+    }
+    updateConfigFieldSetting(accessToken, "enable_public_model_hub", true).then(
+      (data) => {
+        setIsPublicPageModalVisible(true);
+      }
+    );
+  };
+
   const handleOk = () => {
     setIsModalVisible(false);
-
+    setIsPublicPageModalVisible(false);
     setSelectedModel(null);
   };
 
   const handleCancel = () => {
     setIsModalVisible(false);
-
+    setIsPublicPageModalVisible(false);
     setSelectedModel(null);
   };
 
@@ -102,70 +119,118 @@ const ModelHub: React.FC<ModelHubProps> = ({
 
   return (
     <div>
-      <div className="w-full m-2 mt-2 p-8">
-        <div className="relative w-full"></div>
+      {(publicPage && publicPageAllowed) || publicPage == false ? (
+        <div className="w-full m-2 mt-2 p-8">
+          <div className="relative w-full"></div>
 
-        <div className="flex items-center">
-          <Title className="ml-8 text-center ">Model Hub</Title>
-          <Button className="ml-4">
-            <a href="https://forms.gle/W3U4PZpJGFHWtHyA9" target="_blank">
-              ✨ Make Public
-            </a>
-          </Button>
-        </div>
-
-        <div className="grid grid-cols-2 gap-6 sm:grid-cols-3 lg:grid-cols-4">
-          {modelHubData &&
-            modelHubData.map((model: ModelInfo) => (
-              <Card key={model.model_group} className="mt-5 mx-8">
-                <pre className="flex justify-between">
-                  <Title>{model.model_group}</Title>
-                  <Tooltip title={model.model_group}>
-                    <CopyOutlined
-                      onClick={() => copyToClipboard(model.model_group)}
-                      style={{ cursor: "pointer", marginRight: "10px" }}
-                    />
-                  </Tooltip>
-                </pre>
-
-                <div className="my-5">
-                  <Text>Mode: {model.mode}</Text>
-                  <Text>
-                    Supports Function Calling:{" "}
-                    {model?.supports_function_calling == true ? "Yes" : "No"}
-                  </Text>
-                  <Text>
-                    Supports Vision:{" "}
-                    {model?.supports_vision == true ? "Yes" : "No"}
-                  </Text>
-                  <Text>
-                    Max Input Tokens:{" "}
-                    {model?.max_input_tokens ? model?.max_input_tokens : "N/A"}
-                  </Text>
-                  <Text>
-                    Max Output Tokens:{" "}
-                    {model?.max_output_tokens
-                      ? model?.max_output_tokens
-                      : "N/A"}
-                  </Text>
-                </div>
-
-                <div style={{ marginTop: "auto", textAlign: "right" }}>
-                  <a
-                    href="#"
-                    onClick={() => showModal(model)}
-                    style={{ color: "#1890ff", fontSize: "smaller" }}
-                  >
-                    View more <RightOutlined />
+          <div
+            className={`flex ${publicPage ? "justify-between" : "items-center"}`}
+          >
+            <Title className="ml-8 text-center ">Model Hub</Title>
+            {publicPage == false ? (
+              premiumUser ? (
+                <Button className="ml-4" onClick={() => handleMakePublicPage()}>
+                  ✨ Make Public
+                </Button>
+              ) : (
+                <Button className="ml-4">
+                  <a href="https://forms.gle/W3U4PZpJGFHWtHyA9" target="_blank">
+                    ✨ Make Public
                   </a>
-                </div>
-              </Card>
-            ))}
+                </Button>
+              )
+            ) : (
+              <div className="flex justify-between items-center">
+                <p>Filter by key:</p>
+                <Text className="bg-gray-200 pr-2 pl-2 pt-1 pb-1 text-center">{`/ui/model_hub?key=<YOUR_KEY>`}</Text>
+              </div>
+            )}
+          </div>
+
+          <div className="grid grid-cols-2 gap-6 sm:grid-cols-3 lg:grid-cols-4">
+            {modelHubData &&
+              modelHubData.map((model: ModelInfo) => (
+                <Card key={model.model_group} className="mt-5 mx-8">
+                  <pre className="flex justify-between">
+                    <Title>{model.model_group}</Title>
+                    <Tooltip title={model.model_group}>
+                      <CopyOutlined
+                        onClick={() => copyToClipboard(model.model_group)}
+                        style={{ cursor: "pointer", marginRight: "10px" }}
+                      />
+                    </Tooltip>
+                  </pre>
+                  <div className="my-5">
+                    <Text>Mode: {model.mode}</Text>
+                    <Text>
+                      Supports Function Calling:{" "}
+                      {model?.supports_function_calling == true ? "Yes" : "No"}
+                    </Text>
+                    <Text>
+                      Supports Vision:{" "}
+                      {model?.supports_vision == true ? "Yes" : "No"}
+                    </Text>
+                    <Text>
+                      Max Input Tokens:{" "}
+                      {model?.max_input_tokens
+                        ? model?.max_input_tokens
+                        : "N/A"}
+                    </Text>
+                    <Text>
+                      Max Output Tokens:{" "}
+                      {model?.max_output_tokens
+                        ? model?.max_output_tokens
+                        : "N/A"}
+                    </Text>
+                  </div>
+                  <div style={{ marginTop: "auto", textAlign: "right" }}>
+                    <a
+                      href="#"
+                      onClick={() => showModal(model)}
+                      style={{ color: "#1890ff", fontSize: "smaller" }}
+                    >
+                      View more <RightOutlined />
+                    </a>
+                  </div>
+                </Card>
+              ))}
+          </div>
         </div>
-      </div>
+      ) : (
+        <Card className="mx-auto max-w-xl mt-10">
+          <Text className="text-xl text-center mb-2 text-black">
+            Public Model Hub not enabled.
+          </Text>
+          <p className="text-base text-center text-slate-800">
+            Ask your proxy admin to enable this on their Admin UI.
+          </p>
+        </Card>
+      )}
 
       <Modal
-        title="Model Usage"
+        title={"Public Model Hub"}
+        width={600}
+        visible={isPublicPageModalVisible}
+        footer={null}
+        onOk={handleOk}
+        onCancel={handleCancel}
+      >
+        <div className="pt-5 pb-5">
+          <div className="flex justify-between mb-4">
+            <Text className="text-base mr-2">Shareable Link:</Text>
+            <Text className="max-w-sm ml-2 bg-gray-200 pr-2 pl-2 pt-1 pb-1 text-center rounded">{`<proxy_base_url>/ui/model_hub?key=<YOUR_API_KEY>`}</Text>
+          </div>
+          <div className="flex justify-end">
+            <Button onClick={goToPublicModelPage}>See Page</Button>
+          </div>
+        </div>
+      </Modal>
+      <Modal
+        title={
+          selectedModel && selectedModel.model_group
+            ? selectedModel.model_group
+            : "Unknown Model"
+        }
         width={800}
         visible={isModalVisible}
         footer={null}
@@ -174,13 +239,14 @@ const ModelHub: React.FC<ModelHubProps> = ({
       >
         {selectedModel && (
           <div>
-            <p>
-              <strong>Model Name:</strong> {selectedModel.model_group}
+            <p className="mb-4">
+              <strong>Model Information & Usage</strong>
             </p>
 
             <TabGroup>
               <TabList>
                 <Tab>OpenAI Python SDK</Tab>
+                <Tab>Supported OpenAI Params</Tab>
                 <Tab>LlamaIndex</Tab>
                 <Tab>Langchain Py</Tab>
               </TabList>
@@ -208,6 +274,11 @@ print(response)
             `}
                   </SyntaxHighlighter>
                 </TabPanel>
+                <TabPanel>
+                  <SyntaxHighlighter language="python">
+                    {`${selectedModel.supported_openai_params?.map((param) => `${param}\n`).join("")}`}
+                  </SyntaxHighlighter>
+                </TabPanel>
                 <TabPanel>
                   <SyntaxHighlighter language="python">
                     {`

ui/litellm-dashboard/src/components/networking.tsx

@@ -579,17 +579,14 @@ export const modelInfoCall = async (
   }
 };
 
-
-export const modelHubCall = async (
-  accessToken: String,
-  userID: String,
-  userRole: String
-) => {
+export const modelHubCall = async (accessToken: String) => {
   /**
    * Get all models on proxy
    */
   try {
-    let url = proxyBaseUrl ? `${proxyBaseUrl}/model_group/info` : `/model_group/info`;
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/model_group/info`
+      : `/model_group/info`;
 
     //message.info("Requesting model data");
     const response = await fetch(url, {
@@ -617,8 +614,6 @@ export const modelHubCall = async (
   }
 };
 
-
-
 export const modelMetricsCall = async (
   accessToken: String,
   userID: String,
@@ -1870,6 +1865,38 @@ export const getGeneralSettingsCall = async (accessToken: String) => {
   }
 };
 
+export const getConfigFieldSetting = async (
+  accessToken: String,
+  fieldName: string
+) => {
+  try {
+    let url = proxyBaseUrl
+      ? `${proxyBaseUrl}/config/field/info?field_name=${fieldName}`
+      : `/config/field/info?field_name=${fieldName}`;
+
+    //message.info("Requesting model data");
+    const response = await fetch(url, {
+      method: "GET",
+      headers: {
+        Authorization: `Bearer ${accessToken}`,
+        "Content-Type": "application/json",
+      },
+    });
+
+    if (!response.ok) {
+      const errorData = await response.text();
+      throw new Error("Network response was not ok");
+    }
+
+    const data = await response.json();
+    return data;
+    // Handle success - you might want to update some state or UI based on the created key
+  } catch (error) {
+    console.error("Failed to set callbacks:", error);
+    throw error;
+  }
+};
+
 export const updateConfigFieldSetting = async (
   accessToken: String,
   fieldName: string,

ui/litellm-dashboard/src/components/user_dashboard.tsx

@@ -1,6 +1,10 @@
 "use client";
 import React, { useState, useEffect } from "react";
-import { userInfoCall, modelAvailableCall, getTotalSpendCall } from "./networking";
+import {
+  userInfoCall,
+  modelAvailableCall,
+  getTotalSpendCall,
+} from "./networking";
 import { Grid, Col, Card, Text, Title } from "@tremor/react";
 import CreateKey from "./create_key_button";
 import ViewKeyTable from "./view_key_table";
@@ -19,7 +23,6 @@ type UserSpendData = {
   max_budget?: number | null;
 };
 
-
 interface UserDashboardProps {
   userID: string | null;
   userRole: string | null;
@@ -35,8 +38,8 @@ interface UserDashboardProps {
 type TeamInterface = {
   models: any[];
   team_id: null;
-  team_alias: String
-}
+  team_alias: String;
+};
 
 const UserDashboard: React.FC<UserDashboardProps> = ({
   userID,
@@ -63,10 +66,10 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
   const [teamSpend, setTeamSpend] = useState<number | null>(null);
   const [userModels, setUserModels] = useState<string[]>([]);
   const defaultTeam: TeamInterface = {
-    "models": [],
-    "team_alias": "Default Team",
-    "team_id": null
-  }
+    models: [],
+    team_alias: "Default Team",
+    team_id: null,
+  };
   const [selectedTeam, setSelectedTeam] = useState<any | null>(
     teams ? teams[0] : defaultTeam
   );
@@ -137,7 +140,14 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
       } else {
         const fetchData = async () => {
           try {
-            const response = await userInfoCall(accessToken, userID, userRole, false, null, null);
+            const response = await userInfoCall(
+              accessToken,
+              userID,
+              userRole,
+              false,
+              null,
+              null
+            );
             console.log(
               `received teams in user dashboard: ${Object.keys(
                 response
@@ -152,12 +162,12 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
             }
             setKeys(response["keys"]); // Assuming this is the correct path to your data
             setTeams(response["teams"]);
-            const teamsArray = [...response['teams']];
+            const teamsArray = [...response["teams"]];
             if (teamsArray.length > 0) {
-                console.log(`response['teams']: ${teamsArray}`);
-                setSelectedTeam(teamsArray[0]);
+              console.log(`response['teams']: ${teamsArray}`);
+              setSelectedTeam(teamsArray[0]);
             } else {
-                setSelectedTeam(defaultTeam);
+              setSelectedTeam(defaultTeam);
             }
             sessionStorage.setItem(
               "userData" + userID,
@@ -194,22 +204,30 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
         fetchData();
       }
     }
-    
   }, [userID, token, accessToken, keys, userRole]);
 
   useEffect(() => {
     // This code will run every time selectedTeam changes
-    if (keys !== null && selectedTeam !== null && selectedTeam !== undefined) {
+    if (
+      keys !== null &&
+      selectedTeam !== null &&
+      selectedTeam !== undefined &&
+      selectedTeam.team_id !== null
+    ) {
       let sum = 0;
       for (const key of keys) {
-        if (selectedTeam.hasOwnProperty('team_id') && key.team_id !== null && key.team_id === selectedTeam.team_id) {
+        if (
+          selectedTeam.hasOwnProperty("team_id") &&
+          key.team_id !== null &&
+          key.team_id === selectedTeam.team_id
+        ) {
           sum += key.spend;
         }
       }
       setTeamSpend(sum);
     } else if (keys !== null) {
       // sum the keys which don't have team-id set (default team)
-      let sum = 0 
+      let sum = 0;
       for (const key of keys) {
         sum += key.spend;
       }
@@ -245,9 +263,8 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
   }
 
   console.log("inside user dashboard, selected team", selectedTeam);
-  console.log(`teamSpend: ${teamSpend}`)
   return (
-      <div className="w-full mx-4">
+    <div className="w-full mx-4">
       <Grid numItems={1} className="gap-2 p-8 h-[75vh] w-full mt-2">
         <Col numColSpan={1}>
           <ViewUserTeam
@@ -261,8 +278,7 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
             userRole={userRole}
             accessToken={accessToken}
             userSpend={teamSpend}
-            selectedTeam = {selectedTeam ? selectedTeam : null}
-
+            selectedTeam={selectedTeam ? selectedTeam : null}
           />
 
           <ViewKeyTable
@@ -283,11 +299,15 @@ const UserDashboard: React.FC<UserDashboardProps> = ({
             data={keys}
             setData={setKeys}
           />
-          <DashboardTeam teams={teams} setSelectedTeam={setSelectedTeam} userRole={userRole}/>
+          <DashboardTeam
+            teams={teams}
+            setSelectedTeam={setSelectedTeam}
+            userRole={userRole}
+          />
         </Col>
       </Grid>
     </div>
   );
 };
 
-export default UserDashboard;
+export default UserDashboard;