diff --git a/README.md b/README.md index 6200aa47fc2..6a4c738c57b 100644 --- a/README.md +++ b/README.md @@ -121,7 +121,7 @@ response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content # OpenAI Proxy - ([Docs](https://docs.litellm.ai/docs/simple_proxy)) -Track spend across multiple projects/people +Set Budgets & Rate limits across multiple projects The proxy provides: 1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth) @@ -163,7 +163,7 @@ print(response) UI on `/ui` on your proxy server ![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) -Track Spend, Set budgets and create virtual keys for the proxy +Set budgets and rate limits across multiple projects `POST /key/generate` ### Request diff --git a/litellm/__init__.py b/litellm/__init__.py index e75f635f306..3ba33741581 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -165,6 +165,7 @@ s3_callback_params: Optional[Dict] = None generic_logger_headers: Optional[Dict] = None default_key_generate_params: Optional[Dict] = None upperbound_key_generate_params: Optional[Dict] = None +default_user_params: Optional[Dict] = None default_team_settings: Optional[List] = None max_user_budget: Optional[float] = None #### RELIABILITY #### diff --git a/litellm/integrations/langfuse.py b/litellm/integrations/langfuse.py index f7f2634de3c..bb8b62b5079 100644 --- a/litellm/integrations/langfuse.py +++ b/litellm/integrations/langfuse.py @@ -235,6 +235,9 @@ class LangFuseLogger: supports_tags = Version(langfuse.version.__version__) >= Version("2.6.3") supports_prompt = Version(langfuse.version.__version__) >= Version("2.7.3") supports_costs = Version(langfuse.version.__version__) >= Version("2.7.3") + supports_completion_start_time = Version( + langfuse.version.__version__ + ) >= Version("2.7.3") print_verbose(f"Langfuse Layer Logging - logging to langfuse v2 ") @@ -308,6 +311,11 @@ class LangFuseLogger: if output is not None and isinstance(output, str) and level == "ERROR": generation_params["statusMessage"] = output + if supports_completion_start_time: + generation_params["completion_start_time"] = kwargs.get( + "completion_start_time", None + ) + trace.generation(**generation_params) except Exception as e: print(f"Langfuse Layer Error - {traceback.format_exc()}") diff --git a/litellm/llms/huggingface_restapi.py b/litellm/llms/huggingface_restapi.py index 123b8ecbc4e..205bad7eef9 100644 --- a/litellm/llms/huggingface_restapi.py +++ b/litellm/llms/huggingface_restapi.py @@ -575,12 +575,20 @@ class Huggingface(BaseLLM): response = await client.post(url=api_base, json=data, headers=headers) response_json = response.json() if response.status_code != 200: - raise HuggingfaceError( - status_code=response.status_code, - message=response.text, - request=response.request, - response=response, - ) + if "error" in response_json: + raise HuggingfaceError( + status_code=response.status_code, + message=response_json["error"], + request=response.request, + response=response, + ) + else: + raise HuggingfaceError( + status_code=response.status_code, + message=response.text, + request=response.request, + response=response, + ) ## RESPONSE OBJECT return self.convert_to_model_response_object( @@ -595,6 +603,8 @@ class Huggingface(BaseLLM): except Exception as e: if isinstance(e, httpx.TimeoutException): raise HuggingfaceError(status_code=500, message="Request Timeout Error") + elif isinstance(e, HuggingfaceError): + raise e elif response is not None and hasattr(response, "text"): raise HuggingfaceError( status_code=500, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a9fb6d700ad..939dd244a60 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -730,6 +730,8 @@ async def user_api_key_auth( "/user", "/model/info", "/v2/model/info", + "/models", + "/v1/models", ] # check if the current route startswith any of the allowed routes if ( @@ -1758,6 +1760,7 @@ async def generate_key_helper_fn( allowed_cache_controls: Optional[list] = [], permissions: Optional[dict] = {}, model_max_budget: Optional[dict] = {}, + table_name: Optional[Literal["key", "user"]] = None, ): global prisma_client, custom_db_client, user_api_key_cache @@ -1884,8 +1887,10 @@ async def generate_key_helper_fn( table_name="user", update_key_values=update_key_values, ) - if user_id == litellm_proxy_budget_name: - # do not create a key for litellm_proxy_budget_name + if user_id == litellm_proxy_budget_name or ( + table_name is not None and table_name == "user" + ): + # do not create a key for litellm_proxy_budget_name or if table name is set to just 'user' # we only need to ensure this exists in the user table # the LiteLLM_VerificationToken table will increase in size if we don't do this check return key_data @@ -2440,7 +2445,7 @@ async def completion( ) traceback.print_exc() error_traceback = traceback.format_exc() - error_msg = f"{str(e)}\n\n{error_traceback}" + error_msg = f"{str(e)}" raise ProxyException( message=getattr(e, "message", error_msg), type=getattr(e, "type", "None"), @@ -5548,27 +5553,50 @@ async def auth_callback(request: Request): user_id_models: List = [] # User might not be already created on first generation of key - # But if it is, we want its models preferences + # But if it is, we want their models preferences + default_ui_key_values = { + "duration": "1hr", + "key_max_budget": 0.01, + "aliases": {}, + "config": {}, + "spend": 0, + "team_id": "litellm-dashboard", + } + user_defined_values = { + "models": user_id_models, + "user_id": user_id, + "user_email": user_email, + } try: if prisma_client is not None: user_info = await prisma_client.get_data(user_id=user_id, table_name="user") + verbose_proxy_logger.debug( + f"user_info: {user_info}; litellm.default_user_params: {litellm.default_user_params}" + ) if user_info is not None: - user_id_models = getattr(user_info, "models", []) + user_defined_values = { + "models": getattr(user_info, "models", []), + "user_id": getattr(user_info, "user_id", user_id), + "user_email": getattr(user_info, "user_id", user_email), + } + elif litellm.default_user_params is not None and isinstance( + litellm.default_user_params, dict + ): + user_defined_values = { + "models": litellm.default_user_params.get("models", user_id_models), + "user_id": litellm.default_user_params.get("user_id", user_id), + "user_email": litellm.default_user_params.get( + "user_email", user_email + ), + } except Exception as e: pass + verbose_proxy_logger.info( + f"user_defined_values for creating ui key: {user_defined_values}" + ) response = await generate_key_helper_fn( - **{ - "duration": "1hr", - "key_max_budget": 0.01, - "models": user_id_models, - "aliases": {}, - "config": {}, - "spend": 0, - "user_id": user_id, - "team_id": "litellm-dashboard", - "user_email": user_email, - } # type: ignore + **default_ui_key_values, **user_defined_values # type: ignore ) key = response["token"] # type: ignore user_id = response["user_id"] # type: ignore diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 4e08f8fb9ff..9f9774412fd 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -554,6 +554,11 @@ class PrismaClient: f"PrismaClient: find_unique for token: {hashed_token}" ) if query_type == "find_unique": + if token is None: + raise HTTPException( + status_code=400, + detail={"error": f"No token passed in. Token={token}"}, + ) response = await self.db.litellm_verificationtoken.find_unique( where={"token": hashed_token} ) diff --git a/litellm/router.py b/litellm/router.py index 06efd4c34b7..6f33d0b0d5c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -830,8 +830,8 @@ class Router: verbose_router_logger.info( f"litellm.atext_completion(model={model})\033[31m Exception {str(e)}\033[0m" ) - if model_name is not None: - self.fail_calls[model_name] += 1 + if model is not None: + self.fail_calls[model] += 1 raise e def embedding( diff --git a/litellm/tests/test_custom_logger.py b/litellm/tests/test_custom_logger.py index 2747d33e9d9..e1c87f1a324 100644 --- a/litellm/tests/test_custom_logger.py +++ b/litellm/tests/test_custom_logger.py @@ -206,6 +206,7 @@ def test_async_custom_handler_stream(): # test_async_custom_handler_stream() +@pytest.mark.skip(reason="Flaky test") def test_azure_completion_stream(): # [PROD Test] - Do not DELETE # test if completion() + sync custom logger get the same complete stream response diff --git a/litellm/tests/test_streaming.py b/litellm/tests/test_streaming.py index 7baa927ad21..66e8be4cbea 100644 --- a/litellm/tests/test_streaming.py +++ b/litellm/tests/test_streaming.py @@ -1655,6 +1655,202 @@ def test_openai_streaming_and_function_calling(): raise e +# test_azure_streaming_and_function_calling() + + +def test_success_callback_streaming(): + def success_callback(kwargs, completion_response, start_time, end_time): + print( + { + "success": True, + "input": kwargs, + "output": completion_response, + "start_time": start_time, + "end_time": end_time, + } + ) + + litellm.success_callback = [success_callback] + + messages = [{"role": "user", "content": "hello"}] + print("TESTING LITELLM COMPLETION CALL") + response = litellm.completion( + model="j2-light", + messages=messages, + stream=True, + max_tokens=5, + ) + print(response) + + for chunk in response: + print(chunk["choices"][0]) + + +# test_success_callback_streaming() + +#### STREAMING + FUNCTION CALLING ### +from pydantic import BaseModel +from typing import List, Optional + + +class Function(BaseModel): + name: str + arguments: str + + +class ToolCalls(BaseModel): + index: int + id: str + type: str + function: Function + + +class Delta(BaseModel): + role: str + content: Optional[str] + tool_calls: List[ToolCalls] + + +class Choices(BaseModel): + index: int + delta: Delta + logprobs: Optional[str] + finish_reason: Optional[str] + + +class Chunk(BaseModel): + id: str + object: str + created: int + model: str + system_fingerprint: str + choices: List[Choices] + + +def validate_first_streaming_function_calling_chunk(chunk: ModelResponse): + chunk_instance = Chunk(**chunk.model_dump()) + + +### Chunk 1 + + +# { +# "id": "chatcmpl-8vdVjtzxc0JqGjq93NxC79dMp6Qcs", +# "object": "chat.completion.chunk", +# "created": 1708747267, +# "model": "gpt-3.5-turbo-0125", +# "system_fingerprint": "fp_86156a94a0", +# "choices": [ +# { +# "index": 0, +# "delta": { +# "role": "assistant", +# "content": null, +# "tool_calls": [ +# { +# "index": 0, +# "id": "call_oN10vaaC9iA8GLFRIFwjCsN7", +# "type": "function", +# "function": { +# "name": "get_current_weather", +# "arguments": "" +# } +# } +# ] +# }, +# "logprobs": null, +# "finish_reason": null +# } +# ] +# } +class Function2(BaseModel): + arguments: str + + +class ToolCalls2(BaseModel): + index: int + function: Optional[Function2] + + +class Delta2(BaseModel): + tool_calls: List[ToolCalls2] + + +class Choices2(BaseModel): + index: int + delta: Delta2 + logprobs: Optional[str] + finish_reason: Optional[str] + + +class Chunk2(BaseModel): + id: str + object: str + created: int + model: str + system_fingerprint: str + choices: List[Choices2] + + +## Chunk 2 + +# { +# "id": "chatcmpl-8vdVjtzxc0JqGjq93NxC79dMp6Qcs", +# "object": "chat.completion.chunk", +# "created": 1708747267, +# "model": "gpt-3.5-turbo-0125", +# "system_fingerprint": "fp_86156a94a0", +# "choices": [ +# { +# "index": 0, +# "delta": { +# "tool_calls": [ +# { +# "index": 0, +# "function": { +# "arguments": "{\"" +# } +# } +# ] +# }, +# "logprobs": null, +# "finish_reason": null +# } +# ] +# } + + +def validate_second_streaming_function_calling_chunk(chunk: ModelResponse): + chunk_instance = Chunk2(**chunk.model_dump()) + + +class Delta3(BaseModel): + content: Optional[str] = None + role: Optional[str] = None + function_call: Optional[dict] = None + tool_calls: Optional[List] = None + + +class Choices3(BaseModel): + index: int + delta: Delta3 + logprobs: Optional[str] + finish_reason: str + + +class Chunk3(BaseModel): + id: str + object: str + created: int + model: str + system_fingerprint: str + choices: List[Choices3] + + +def validate_final_streaming_function_calling_chunk(chunk: ModelResponse): + chunk_instance = Chunk3(**chunk.model_dump()) + + def test_azure_streaming_and_function_calling(): tools = [ { @@ -1690,6 +1886,7 @@ def test_azure_streaming_and_function_calling(): ) # Add any assertions here to check the response for idx, chunk in enumerate(response): + print(f"chunk: {chunk}") if idx == 0: assert ( chunk.choices[0].delta.tool_calls[0].function.arguments is not None @@ -1697,40 +1894,69 @@ def test_azure_streaming_and_function_calling(): assert isinstance( chunk.choices[0].delta.tool_calls[0].function.arguments, str ) + validate_first_streaming_function_calling_chunk(chunk=chunk) + elif idx == 1: + validate_second_streaming_function_calling_chunk(chunk=chunk) + elif chunk.choices[0].finish_reason is not None: # last chunk + validate_final_streaming_function_calling_chunk(chunk=chunk) + except Exception as e: pytest.fail(f"Error occurred: {e}") raise e -# test_azure_streaming_and_function_calling() - - -def test_success_callback_streaming(): - def success_callback(kwargs, completion_response, start_time, end_time): - print( - { - "success": True, - "input": kwargs, - "output": completion_response, - "start_time": start_time, - "end_time": end_time, - } +@pytest.mark.asyncio +async def test_azure_astreaming_and_function_calling(): + tools = [ + { + "type": "function", + "function": { + "name": "get_current_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + }, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, + }, + "required": ["location"], + }, + }, + } + ] + messages = [{"role": "user", "content": "What is the weather like in Boston?"}] + try: + response = await litellm.acompletion( + model="azure/gpt-4-nov-release", + tools=tools, + tool_choice="auto", + messages=messages, + stream=True, + api_base=os.getenv("AZURE_FRANCE_API_BASE"), + api_key=os.getenv("AZURE_FRANCE_API_KEY"), + api_version="2024-02-15-preview", ) + # Add any assertions here to check the response + idx = 0 + async for chunk in response: + print(f"chunk: {chunk}") + if idx == 0: + assert ( + chunk.choices[0].delta.tool_calls[0].function.arguments is not None + ) + assert isinstance( + chunk.choices[0].delta.tool_calls[0].function.arguments, str + ) + validate_first_streaming_function_calling_chunk(chunk=chunk) + elif idx == 1: + validate_second_streaming_function_calling_chunk(chunk=chunk) + elif chunk.choices[0].finish_reason is not None: # last chunk + validate_final_streaming_function_calling_chunk(chunk=chunk) + idx += 1 - litellm.success_callback = [success_callback] - - messages = [{"role": "user", "content": "hello"}] - print("TESTING LITELLM COMPLETION CALL") - response = litellm.completion( - model="j2-light", - messages=messages, - stream=True, - max_tokens=5, - ) - print(response) - - for chunk in response: - print(chunk["choices"][0]) - - -# test_success_callback_streaming() + except Exception as e: + pytest.fail(f"Error occurred: {e}") + raise e diff --git a/litellm/utils.py b/litellm/utils.py index c67baef3c55..38ca669fc74 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -376,11 +376,9 @@ class StreamingChoices(OpenAIObject): self.delta = delta else: self.delta = Delta() - - if logprobs is not None: - self.logprobs = logprobs if enhancements is not None: self.enhancements = enhancements + self.logprobs = logprobs def __contains__(self, key): # Define custom behavior for the 'in' operator @@ -820,6 +818,8 @@ class Logging: ## DYNAMIC LANGFUSE KEYS ## self.langfuse_public_key = langfuse_public_key self.langfuse_secret = langfuse_secret + ## TIME TO FIRST TOKEN LOGGING ## + self.completion_start_time: Optional[datetime.datetime] = None def update_environment_variables( self, model, user, optional_params, litellm_params, **additional_params @@ -840,6 +840,7 @@ class Logging: "user": user, "call_type": str(self.call_type), "litellm_call_id": self.litellm_call_id, + "completion_start_time": self.completion_start_time, **self.optional_params, **additional_params, } @@ -1069,6 +1070,11 @@ class Logging: start_time = self.start_time if end_time is None: end_time = datetime.datetime.now() + if self.completion_start_time is None: + self.completion_start_time = end_time + self.model_call_details["completion_start_time"] = ( + self.completion_start_time + ) self.model_call_details["log_event_type"] = "successful_api_call" self.model_call_details["end_time"] = end_time self.model_call_details["cache_hit"] = cache_hit @@ -1358,7 +1364,7 @@ class Logging: f"is complete_streaming_response in kwargs: {kwargs.get('complete_streaming_response', None)}" ) if complete_streaming_response is None: - break + continue else: print_verbose("reaches langfuse for streaming logging!") result = kwargs["complete_streaming_response"] @@ -8629,6 +8635,10 @@ class CustomStreamWrapper: model_response.choices[0].finish_reason = response_obj[ "finish_reason" ] + if response_obj.get("original_chunk", None) is not None: + model_response.system_fingerprint = getattr( + response_obj["original_chunk"], "system_fingerprint", None + ) if response_obj["logprobs"] is not None: model_response.choices[0].logprobs = response_obj["logprobs"] diff --git a/pyproject.toml b/pyproject.toml index 33280817797..4aa154f8622 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.26.14.dev1" +version = "1.27.1" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -74,7 +74,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.26.14.dev1" +version = "1.27.1" version_files = [ "pyproject.toml:^version" ] diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index fefb4243f85..0d15c598ba0 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -97,10 +97,10 @@ async def test_chat_completion_old_key(): """ async with aiohttp.ClientSession() as session: try: - key = "sk-yNXvlRO4SxIGG0XnRMYxTw" + key = "sk-ecMXHujzUtKCvHcwacdaTw" await chat_completion(session=session, key=key) except Exception as e: - key = "sk-2KV0sAElLQqMpLZXdNf3yw" # try diff db key (in case db url is for the other db) + key = "sk-ecMXHujzUtKCvHcwacdaTw" # try diff db key (in case db url is for the other db) await chat_completion(session=session, key=key) diff --git a/ui/litellm-dashboard/src/components/chat_ui.tsx b/ui/litellm-dashboard/src/components/chat_ui.tsx index edab67b2bad..102354222f2 100644 --- a/ui/litellm-dashboard/src/components/chat_ui.tsx +++ b/ui/litellm-dashboard/src/components/chat_ui.tsx @@ -1,17 +1,26 @@ import React, { useState, useEffect } from "react"; import ReactMarkdown from "react-markdown"; -import { Card, Title, Table, TableHead, TableRow, TableCell, TableBody, Grid, Tab, - TabGroup, - TabList, - TabPanel, - Metric, - Select, - SelectItem, - TabPanels, } from "@tremor/react"; -import { modelInfoCall } from "./networking"; +import { + Card, + Title, + Table, + TableHead, + TableRow, + TableCell, + TableBody, + Grid, + Tab, + TabGroup, + TabList, + TabPanel, + Metric, + Select, + SelectItem, + TabPanels, +} from "@tremor/react"; +import { modelAvailableCall } from "./networking"; import openai from "openai"; -import { Prism as SyntaxHighlighter } from 'react-syntax-highlighter'; - +import { Prism as SyntaxHighlighter } from "react-syntax-highlighter"; interface ChatUIProps { accessToken: string | null; @@ -20,12 +29,19 @@ interface ChatUIProps { userID: string | null; } -async function generateModelResponse(inputMessage: string, updateUI: (chunk: string) => void, selectedModel: string, accessToken: string) { - // base url should be the current base_url - const isLocal = process.env.NODE_ENV === "development"; - console.log("isLocal:", isLocal); - const proxyBaseUrl = isLocal ? "http://localhost:4000" : window.location.origin; - const client = new openai.OpenAI({ +async function generateModelResponse( + inputMessage: string, + updateUI: (chunk: string) => void, + selectedModel: string, + accessToken: string +) { + // base url should be the current base_url + const isLocal = process.env.NODE_ENV === "development"; + console.log("isLocal:", isLocal); + const proxyBaseUrl = isLocal + ? "http://localhost:4000" + : window.location.origin; + const client = new openai.OpenAI({ apiKey: accessToken, // Replace with your OpenAI API key baseURL: proxyBaseUrl, // Replace with your OpenAI API base URL dangerouslyAllowBrowser: true, // using a temporary litellm proxy key @@ -36,7 +52,7 @@ async function generateModelResponse(inputMessage: string, updateUI: (chunk: str stream: true, messages: [ { - role: 'user', + role: "user", content: inputMessage, }, ], @@ -50,138 +66,166 @@ async function generateModelResponse(inputMessage: string, updateUI: (chunk: str } } -const ChatUI: React.FC = ({ accessToken, token, userRole, userID }) => { - const [inputMessage, setInputMessage] = useState(""); - const [chatHistory, setChatHistory] = useState([]); - const [selectedModel, setSelectedModel] = useState(undefined); - const [modelInfo, setModelInfo] = useState(null); // Declare modelInfo at the component level +const ChatUI: React.FC = ({ + accessToken, + token, + userRole, + userID, +}) => { + const [inputMessage, setInputMessage] = useState(""); + const [chatHistory, setChatHistory] = useState([]); + const [selectedModel, setSelectedModel] = useState( + undefined + ); + const [modelInfo, setModelInfo] = useState(null); // Declare modelInfo at the component level - useEffect(() => { - if (!accessToken || !token || !userRole || !userID) { - return; - } - // Fetch model info and set the default selected model - const fetchModelInfo = async () => { - const fetchedModelInfo = await modelInfoCall(accessToken, userID, userRole); - console.log("model_info:", fetchedModelInfo); - - if (fetchedModelInfo?.data.length > 0) { - setModelInfo(fetchedModelInfo); - setSelectedModel(fetchedModelInfo.data[0].model_name); - } - }; - - fetchModelInfo(); - }, [accessToken, userID, userRole]); - - const updateUI = (role: string, chunk: string) => { - setChatHistory((prevHistory) => { - const lastMessage = prevHistory[prevHistory.length - 1]; - - if (lastMessage && lastMessage.role === role) { - return [ - ...prevHistory.slice(0, prevHistory.length - 1), - { role, content: lastMessage.content + chunk }, - ]; - } else { - return [...prevHistory, { role, content: chunk }]; - } - }); - }; - - const handleSendMessage = async () => { - if (inputMessage.trim() === "") return; - - if (!accessToken || !token || !userRole || !userID) { - return; + useEffect(() => { + if (!accessToken || !token || !userRole || !userID) { + return; + } + // Fetch model info and set the default selected model + const fetchModelInfo = async () => { + const fetchedAvailableModels = await modelAvailableCall( + accessToken, + userID, + userRole + ); + console.log("model_info:", fetchedAvailableModels); + + if (fetchedAvailableModels?.data.length > 0) { + setModelInfo(fetchedAvailableModels.data); + setSelectedModel(fetchedAvailableModels.data[0].id); } - - setChatHistory((prevHistory) => [ - ...prevHistory, - { role: "user", content: inputMessage }, - ]); - - try { - if (selectedModel) { - await generateModelResponse(inputMessage, (chunk) => updateUI("assistant", chunk), selectedModel, accessToken); - } - } catch (error) { - console.error("Error fetching model response", error); - updateUI("assistant", "Error fetching model response"); - } - - setInputMessage(""); }; - - return ( -
- - + + fetchModelInfo(); + }, [accessToken, userID, userRole]); + + const updateUI = (role: string, chunk: string) => { + setChatHistory((prevHistory) => { + const lastMessage = prevHistory[prevHistory.length - 1]; + + if (lastMessage && lastMessage.role === role) { + return [ + ...prevHistory.slice(0, prevHistory.length - 1), + { role, content: lastMessage.content + chunk }, + ]; + } else { + return [...prevHistory, { role, content: chunk }]; + } + }); + }; + + const handleSendMessage = async () => { + if (inputMessage.trim() === "") return; + + if (!accessToken || !token || !userRole || !userID) { + return; + } + + setChatHistory((prevHistory) => [ + ...prevHistory, + { role: "user", content: inputMessage }, + ]); + + try { + if (selectedModel) { + await generateModelResponse( + inputMessage, + (chunk) => updateUI("assistant", chunk), + selectedModel, + accessToken + ); + } + } catch (error) { + console.error("Error fetching model response", error); + updateUI("assistant", "Error fetching model response"); + } + + setInputMessage(""); + }; + + return ( +
+ + - Chat - API Reference + Chat + API Reference - -
- - setSelectedModel(e.target.value)} + > + {/* Populate dropdown options from available models */} + {modelInfo?.map((element: { id: string }) => ( + + ))} + +
+ + + + + Chat + + + + + {chatHistory.map((message, index) => ( + + {`${message.role}: ${message.content}`} + + ))} + +
+
+
+ setInputMessage(e.target.value)} + className="flex-1 p-2 border rounded-md mr-2" + placeholder="Type your message..." + /> +
- - - - - Chat - - - - - {chatHistory.map((message, index) => ( - - {`${message.role}: ${message.content}`} - - ))} - -
-
-
- setInputMessage(e.target.value)} - className="flex-1 p-2 border rounded-md mr-2" - placeholder="Type your message..." - /> - -
-
- - - - - OpenAI Python SDK - LlamaIndex - Langchain Py - - - - - - {` + Send + +
+
+ + + + + OpenAI Python SDK + LlamaIndex + Langchain Py + + + + + {` import openai client = openai.OpenAI( api_key="your_api_key", @@ -208,12 +252,11 @@ response = client.chat.completions.create( print(response) `} - - - - - - {` + + + + + {` import os, dotenv from llama_index.llms import AzureOpenAI @@ -245,12 +288,11 @@ response = query_engine.query("What did the author do growing up?") print(response) `} - - - - - - {` + + + + + {` from langchain.chat_models import ChatOpenAI from langchain.prompts.chat import ( ChatPromptTemplate, @@ -286,20 +328,17 @@ response = chat(messages) print(response) `} - - - - - - + + + + + - - - -
- ); - }; + + + + + ); +}; - - - export default ChatUI; \ No newline at end of file +export default ChatUI; diff --git a/ui/litellm-dashboard/src/components/create_key_button.tsx b/ui/litellm-dashboard/src/components/create_key_button.tsx index 113952ea82d..03ef9a89e16 100644 --- a/ui/litellm-dashboard/src/components/create_key_button.tsx +++ b/ui/litellm-dashboard/src/components/create_key_button.tsx @@ -1,10 +1,17 @@ - "use client"; import React, { useState, useEffect, useRef } from "react"; import { Button, TextInput, Grid, Col } from "@tremor/react"; import { Card, Metric, Text } from "@tremor/react"; -import { Button as Button2, Modal, Form, Input, InputNumber, Select, message } from "antd"; +import { + Button as Button2, + Modal, + Form, + Input, + InputNumber, + Select, + message, +} from "antd"; import { keyCreateCall } from "./networking"; const { Option } = Select; @@ -50,12 +57,11 @@ const CreateKey: React.FC = ({ setApiKey(response["key"]); message.success("API Key Created"); form.resetFields(); - localStorage.removeItem("userData" + userID) + localStorage.removeItem("userData" + userID); } catch (error) { console.error("Error creating the key:", error); } }; - return (
@@ -70,98 +76,70 @@ const CreateKey: React.FC = ({ onOk={handleOk} onCancel={handleCancel} > -
- {userRole === 'App Owner' || userRole === 'Admin' ? ( + + {userRole === "App Owner" || userRole === "Admin" ? ( <> - - - - - - - - - - - - - - - - - - - - - - - - - - - ) : ( - <> + + + + + + + + + + + + + + + - - - - - + label="Requests per minute Limit (RPM)" + name="rpm_limit" + > + + + + + + + + + + ) : ( + <> + + + + + + - - - - - ) - } -
- - Create Key - -
+ + + + + )} +
+ Create Key +
{apiKey && ( @@ -173,22 +151,22 @@ const CreateKey: React.FC = ({ footer={null} > - -

- Please save this secret key somewhere safe and accessible. For - security reasons, you will not be able to view it again{" "} - through your LiteLLM account. If you lose this secret key, you will - need to generate a new one. -

- - - {apiKey != null ? ( - API Key: {apiKey} - ) : ( - Key being created, this might take 30s - )} - - + +

+ Please save this secret key somewhere safe and accessible. For + security reasons, you will not be able to view it again{" "} + through your LiteLLM account. If you lose this secret key, you + will need to generate a new one. +

+ + + {apiKey != null ? ( + API Key: {apiKey} + ) : ( + Key being created, this might take 30s + )} + + )}
diff --git a/ui/litellm-dashboard/src/components/create_user_button.tsx b/ui/litellm-dashboard/src/components/create_user_button.tsx index b58674d12fa..94ddcabb04e 100644 --- a/ui/litellm-dashboard/src/components/create_user_button.tsx +++ b/ui/litellm-dashboard/src/components/create_user_button.tsx @@ -1,7 +1,7 @@ import React, { useState, useEffect } from "react"; import { Button, Modal, Form, Input, message, Select, InputNumber } from "antd"; import { Button as Button2 } from "@tremor/react"; -import { userCreateCall, modelInfoCall } from "./networking"; +import { userCreateCall, modelAvailableCall } from "./networking"; const { Option } = Select; interface CreateuserProps { @@ -14,18 +14,22 @@ const Createuser: React.FC = ({ userID, accessToken }) => { const [isModalVisible, setIsModalVisible] = useState(false); const [apiuser, setApiuser] = useState(null); const [userModels, setUserModels] = useState([]); - + // get all models useEffect(() => { const fetchData = async () => { try { const userRole = "any"; // You may need to get the user role dynamically - const modelDataResponse = await modelInfoCall(accessToken, userID, userRole); + const modelDataResponse = await modelAvailableCall( + accessToken, + userID, + userRole + ); // Assuming modelDataResponse.data contains an array of model objects with a 'model_name' property const availableModels = []; for (let i = 0; i < modelDataResponse.data.length; i++) { - const model = modelDataResponse.data[i]; - availableModels.push(model.model_name); + const model = modelDataResponse.data[i]; + availableModels.push(model.id); } console.log("Model data response:", modelDataResponse.data); console.log("Available models:", availableModels); @@ -79,77 +83,51 @@ const Createuser: React.FC = ({ userID, accessToken }) => { onOk={handleOk} onCancel={handleCancel} > -
- + + - - - - + + + + - - + - - - - - - - - - - - - - - - - - - - - -
- -
+ + + + + + + + + + + + + + + +
+ +
{apiuser && ( @@ -162,11 +140,15 @@ const Createuser: React.FC = ({ userID, accessToken }) => { >

Please save this secret user somewhere safe and accessible. For - security reasons, you will not be able to view it again through - your LiteLLM account. If you lose this secret user, you will need to - generate a new one. + security reasons, you will not be able to view it again{" "} + through your LiteLLM account. If you lose this secret user, you will + need to generate a new one. +

+

+ {apiuser != null + ? `API user: ${apiuser}` + : "User being created, this might take 30s"}

-

{apiuser != null ? `API user: ${apiuser}` : "User being created, this might take 30s"}

)} diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 250515eeb84..92b285848eb 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -69,7 +69,6 @@ export const keyCreateCall = async ( } }; - export const userCreateCall = async ( accessToken: string, userID: string, @@ -133,7 +132,6 @@ export const userCreateCall = async ( } }; - export const keyDeleteCall = async (accessToken: String, user_key: String) => { try { const url = proxyBaseUrl ? `${proxyBaseUrl}/key/delete` : `/key/delete`; @@ -207,13 +205,14 @@ export const userInfoCall = async ( } }; - - export const modelInfoCall = async ( accessToken: String, userID: String, userRole: String ) => { + /** + * Get all models on proxy + */ try { let url = proxyBaseUrl ? `${proxyBaseUrl}/v2/model/info` : `/v2/model/info`; @@ -242,6 +241,42 @@ export const modelInfoCall = async ( } }; +export const modelAvailableCall = async ( + accessToken: String, + userID: String, + userRole: String +) => { + /** + * Get all the models user has access to + */ + try { + let url = proxyBaseUrl ? `${proxyBaseUrl}/models` : `/models`; + + message.info("Requesting model data"); + const response = await fetch(url, { + method: "GET", + headers: { + Authorization: `Bearer ${accessToken}`, + "Content-Type": "application/json", + }, + }); + + if (!response.ok) { + const errorData = await response.text(); + message.error(errorData); + throw new Error("Network response was not ok"); + } + + const data = await response.json(); + message.info("Received model data"); + return data; + // Handle success - you might want to update some state or UI based on the created key + } catch (error) { + console.error("Failed to create key:", error); + throw error; + } +}; + export const keySpendLogsCall = async (accessToken: String, token: String) => { try { const url = proxyBaseUrl ? `${proxyBaseUrl}/spend/logs` : `/spend/logs`; @@ -363,12 +398,16 @@ export const spendUsersCall = async (accessToken: String, userID: String) => { } }; - - - -export const userRequestModelCall = async (accessToken: String, model: String, UserID: String, justification: String) => { +export const userRequestModelCall = async ( + accessToken: String, + model: String, + UserID: String, + justification: String +) => { try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/user/request_model` : `/user/request_model`; + const url = proxyBaseUrl + ? `${proxyBaseUrl}/user/request_model` + : `/user/request_model`; const response = await fetch(url, { method: "POST", headers: { @@ -398,10 +437,11 @@ export const userRequestModelCall = async (accessToken: String, model: String, U } }; - export const userGetRequesedtModelsCall = async (accessToken: String) => { try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/user/get_requests` : `/user/get_requests`; + const url = proxyBaseUrl + ? `${proxyBaseUrl}/user/get_requests` + : `/user/get_requests`; console.log("in userGetRequesedtModelsCall:", url); const response = await fetch(url, { method: "GET", @@ -425,4 +465,4 @@ export const userGetRequesedtModelsCall = async (accessToken: String) => { console.error("Failed to get requested models:", error); throw error; } -}; \ No newline at end of file +}; diff --git a/ui/litellm-dashboard/src/components/user_dashboard.tsx b/ui/litellm-dashboard/src/components/user_dashboard.tsx index a85d70a8325..ad007086402 100644 --- a/ui/litellm-dashboard/src/components/user_dashboard.tsx +++ b/ui/litellm-dashboard/src/components/user_dashboard.tsx @@ -1,6 +1,6 @@ "use client"; import React, { useState, useEffect } from "react"; -import { userInfoCall, modelInfoCall } from "./networking"; +import { userInfoCall, modelAvailableCall } from "./networking"; import { Grid, Col, Card, Text } from "@tremor/react"; import CreateKey from "./create_key_button"; import ViewKeyTable from "./view_key_table"; @@ -48,10 +48,9 @@ const UserDashboard: React.FC = ({ const token = searchParams.get("token"); const [accessToken, setAccessToken] = useState(null); const [userModels, setUserModels] = useState([]); - // check if window is not undefined if (typeof window !== "undefined") { - window.addEventListener('beforeunload', function() { + window.addEventListener("beforeunload", function () { // Clear session storage sessionStorage.clear(); }); @@ -78,7 +77,6 @@ const UserDashboard: React.FC = ({ // Moved useEffect inside the component and used a condition to run fetch only if the params are available useEffect(() => { - if (token) { const decoded = jwtDecode(token) as { [key: string]: any }; if (decoded) { @@ -109,32 +107,39 @@ const UserDashboard: React.FC = ({ const cachedUserModels = sessionStorage.getItem("userModels" + userID); if (cachedUserModels) { setUserModels(JSON.parse(cachedUserModels)); - } else { const fetchData = async () => { try { const response = await userInfoCall(accessToken, userID, userRole); setUserSpendData(response["user_info"]); setData(response["keys"]); // Assuming this is the correct path to your data - sessionStorage.setItem("userData" + userID, JSON.stringify(response["keys"])); + sessionStorage.setItem( + "userData" + userID, + JSON.stringify(response["keys"]) + ); sessionStorage.setItem( "userSpendData" + userID, JSON.stringify(response["user_info"]) ); - const model_info = await modelInfoCall(accessToken, userID, userRole); - console.log("model_info:", model_info); + const model_available = await modelAvailableCall( + accessToken, + userID, + userRole + ); // loop through model_info["data"] and create an array of element.model_name - let available_model_names = model_info["data"].filter((element: { model_name: string; user_access: boolean }) => element.user_access === true).map((element: { model_name: string; }) => element.model_name); + let available_model_names = model_available["data"].map( + (element: { id: string }) => element.id + ); console.log("available_model_names:", available_model_names); setUserModels(available_model_names); console.log("userModels:", userModels); - sessionStorage.setItem("userModels" + userID, JSON.stringify(available_model_names)); - - - + sessionStorage.setItem( + "userModels" + userID, + JSON.stringify(available_model_names) + ); } catch (error) { console.error("There was an error fetching the data", error); // Optionally, update your UI to reflect the error state here as well