From 2f11b92698a1e7d07abd139d8fe934a022f0640f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 23 Jan 2024 14:15:19 -0800 Subject: [PATCH 1/7] v0 view spend logs --- ui/admin.py | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/ui/admin.py b/ui/admin.py index 2d823d85d7f..80eca7d3cf8 100644 --- a/ui/admin.py +++ b/ui/admin.py @@ -178,6 +178,42 @@ def list_models(): ) +def usage_stats(): + import streamlit as st + import requests + + # Check if the necessary configuration is available + if ( + st.session_state.get("api_url", None) is not None + and st.session_state.get("proxy_key", None) is not None + ): + # Make the GET request + try: + complete_url = "" + if isinstance(st.session_state["api_url"], str) and st.session_state[ + "api_url" + ].endswith("/"): + complete_url = f"{st.session_state['api_url']}models" + else: + complete_url = f"{st.session_state['api_url']}/models" + response = requests.get( + complete_url, + headers={"Authorization": f"Bearer {st.session_state['proxy_key']}"}, + ) + # Check if the request was successful + if response.status_code == 200: + models = response.json() + st.write(models) # or st.json(models) to pretty print the JSON + else: + st.error(f"Failed to get models. Status code: {response.status_code}") + except Exception as e: + st.error(f"An error occurred while requesting models: {e}") + else: + st.warning( + "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + ) + + def create_key(): import streamlit as st import json, requests, uuid @@ -338,6 +374,7 @@ def admin_page(is_admin="NOT_GIVEN"): "Add Models", "List Models", "Create Key", + "Usage Stats", "End-User Auth", ), ) @@ -369,6 +406,8 @@ def admin_page(is_admin="NOT_GIVEN"): list_models() elif page == "Create Key": create_key() + elif page == "Usage Stats": + usage_stats() admin_page() From a2da9c30fbf7c0232944d0e3f8d7c7129f6052c7 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 23 Jan 2024 15:10:10 -0800 Subject: [PATCH 2/7] (feat) add /spend/keys endpoint --- litellm/proxy/proxy_server.py | 24 ++++++++++++++++++++++++ litellm/proxy/utils.py | 4 ++++ 2 files changed, 28 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a1790f49cea..a69e379583c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2305,6 +2305,30 @@ async def info_key_fn( ) +@router.get( + "/spend/keys", + tags=["Budget & Spend Tracking"], + dependencies=[Depends(user_api_key_auth)], +) +async def spend_key_fn(): + global prisma_client + try: + if prisma_client is None: + raise Exception( + f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" + ) + + key_info = await prisma_client.get_data(table_name="key", query_type="find_all") + + return key_info + + except Exception as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": str(e)}, + ) + + #### USER MANAGEMENT #### @router.post( "/user/new", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index c19137d5718..2a5495919de 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -391,6 +391,10 @@ class PrismaClient: for r in response: if isinstance(r.expires, datetime): r.expires = r.expires.isoformat() + elif query_type == "find_all": + response = await self.db.litellm_verificationtoken.find_many( + order={"spend": "desc"}, + ) print_verbose(f"PrismaClient: response={response}") if response is not None: return response From 1158ff49952038ecaa7280a0f056294c9890d987 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 23 Jan 2024 15:58:14 -0800 Subject: [PATCH 3/7] (feat) use cli args to start streamlit --- litellm/proxy/admin_ui.py | 4 +- ui/admin.py | 101 ++++++++++++++++++++++++++++---------- 2 files changed, 78 insertions(+), 27 deletions(-) diff --git a/litellm/proxy/admin_ui.py b/litellm/proxy/admin_ui.py index d50d8be9087..c72cd88f0bb 100644 --- a/litellm/proxy/admin_ui.py +++ b/litellm/proxy/admin_ui.py @@ -98,7 +98,7 @@ def list_models(): st.error(f"An error occurred while requesting models: {e}") else: st.warning( - "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + f"Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page. Currently set Proxy Endpoint: {st.session_state.get('api_url', None)} and Proxy Key: {st.session_state.get('proxy_key', None)}" ) @@ -151,7 +151,7 @@ def create_key(): raise e else: st.warning( - "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + f"Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page. Currently set Proxy Endpoint: {st.session_state.get('api_url', None)} and Proxy Key: {st.session_state.get('proxy_key', None)}" ) diff --git a/ui/admin.py b/ui/admin.py index 80eca7d3cf8..59fa034e115 100644 --- a/ui/admin.py +++ b/ui/admin.py @@ -6,6 +6,9 @@ from dotenv import load_dotenv load_dotenv() import streamlit as st import base64, os, json, uuid, requests +import pandas as pd +import plotly.express as px +import click # Replace your_base_url with the actual URL where the proxy auth app is hosted your_base_url = os.getenv("BASE_URL") # Example base URL @@ -75,7 +78,7 @@ def add_new_model(): and st.session_state.get("proxy_key", None) is None ): st.warning( - "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + f"Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page. Currently set Proxy Endpoint: {st.session_state.get('api_url', None)} and Proxy Key: {st.session_state.get('proxy_key', None)}" ) model_name = st.text_input( @@ -174,11 +177,11 @@ def list_models(): st.error(f"An error occurred while requesting models: {e}") else: st.warning( - "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + f"Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page. Currently set Proxy Endpoint: {st.session_state.get('api_url', None)} and Proxy Key: {st.session_state.get('proxy_key', None)}" ) -def usage_stats(): +def spend_per_key(): import streamlit as st import requests @@ -193,27 +196,50 @@ def usage_stats(): if isinstance(st.session_state["api_url"], str) and st.session_state[ "api_url" ].endswith("/"): - complete_url = f"{st.session_state['api_url']}models" + complete_url = f"{st.session_state['api_url']}/spend/keys" else: - complete_url = f"{st.session_state['api_url']}/models" + complete_url = f"{st.session_state['api_url']}/spend/keys" response = requests.get( complete_url, headers={"Authorization": f"Bearer {st.session_state['proxy_key']}"}, ) # Check if the request was successful if response.status_code == 200: - models = response.json() - st.write(models) # or st.json(models) to pretty print the JSON + spend_per_key = response.json() + # Create DataFrame + spend_df = pd.DataFrame(spend_per_key) + + # Display the spend per key as a graph + st.write("Spend per Key - Top 10:") + top_10_df = spend_df.nlargest(10, "spend") + fig = px.bar( + top_10_df, + x="token", + y="spend", + title="Top 10 Spend per Key", + height=500, # Adjust the height + width=800, # Adjust the width) + ) + st.plotly_chart(fig) + + # Display the spend per key as a table + st.write("Spend per Key - Full Table:") + st.table(spend_df) + else: st.error(f"Failed to get models. Status code: {response.status_code}") except Exception as e: st.error(f"An error occurred while requesting models: {e}") else: st.warning( - "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + f"Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page. Currently set Proxy Endpoint: {st.session_state.get('api_url', None)} and Proxy Key: {st.session_state.get('proxy_key', None)}" ) +def spend_per_user(): + pass + + def create_key(): import streamlit as st import json, requests, uuid @@ -223,7 +249,7 @@ def create_key(): and st.session_state.get("proxy_key", None) is None ): st.warning( - "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + f"Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page. Currently set Proxy Endpoint: {st.session_state.get('api_url', None)} and Proxy Key: {st.session_state.get('proxy_key', None)}" ) duration = st.text_input("Duration - Can be in (h,m,s)", placeholder="1h") @@ -271,7 +297,7 @@ def update_config(): and st.session_state.get("proxy_key", None) is None ): st.warning( - "Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page." + f"Please configure the Proxy Endpoint and Proxy Key on the Proxy Setup page. Currently set Proxy Endpoint: {st.session_state.get('api_url', None)} and Proxy Key: {st.session_state.get('proxy_key', None)}" ) st.markdown("#### Alerting") @@ -360,12 +386,16 @@ def update_config(): raise e -def admin_page(is_admin="NOT_GIVEN"): +def admin_page(is_admin="NOT_GIVEN", input_api_url=None, input_proxy_key=None): # Display the form for the admin to set the proxy URL and allowed email subdomain + st.set_page_config( + layout="wide", # Use "wide" layout for more space + ) st.header("Admin Configuration") st.session_state.setdefault("is_admin", is_admin) # Add a navigation sidebar st.sidebar.title("Navigation") + page = st.sidebar.radio( "Go to", ( @@ -374,23 +404,31 @@ def admin_page(is_admin="NOT_GIVEN"): "Add Models", "List Models", "Create Key", - "Usage Stats", + "View Spend Per Key", + "View Spend Per User", "End-User Auth", ), ) # Display different pages based on navigation selection if page == "Connect to Proxy": # Use text inputs with intermediary variables - input_api_url = st.text_input( - "Proxy Endpoint", - value=st.session_state.get("api_url", ""), - placeholder="http://0.0.0.0:8000", - ) - input_proxy_key = st.text_input( - "Proxy Key", - value=st.session_state.get("proxy_key", ""), - placeholder="sk-...", - ) + if input_api_url is None: + input_api_url = st.text_input( + "Proxy Endpoint", + value=st.session_state.get("api_url", ""), + placeholder="http://0.0.0.0:8000", + ) + else: + st.session_state["api_url"] = input_api_url + + if input_proxy_key is None: + input_proxy_key = st.text_input( + "Proxy Key", + value=st.session_state.get("proxy_key", ""), + placeholder="sk-...", + ) + else: + st.session_state["proxy_key"] = input_proxy_key # When the "Save" button is clicked, update the session state if st.button("Save"): st.session_state["api_url"] = input_api_url @@ -406,8 +444,21 @@ def admin_page(is_admin="NOT_GIVEN"): list_models() elif page == "Create Key": create_key() - elif page == "Usage Stats": - usage_stats() + elif page == "View Spend Per Key": + spend_per_key() + elif page == "View Spend Per User": + spend_per_user() -admin_page() +# admin_page() + + +@click.command() +@click.option("--proxy_endpoint", type=str, help="Proxy Endpoint") +@click.option("--proxy_master_key", type=str, help="Proxy Master Key") +def main(proxy_endpoint, proxy_master_key): + admin_page(input_api_url=proxy_endpoint, input_proxy_key=proxy_master_key) + + +if __name__ == "__main__": + main() From e723df30f35ef3c5931b67a9da908da3667ea193 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 23 Jan 2024 16:14:39 -0800 Subject: [PATCH 4/7] (feat) ui improvements --- ui/admin.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/ui/admin.py b/ui/admin.py index 59fa034e115..674bba7fe0d 100644 --- a/ui/admin.py +++ b/ui/admin.py @@ -210,15 +210,16 @@ def spend_per_key(): spend_df = pd.DataFrame(spend_per_key) # Display the spend per key as a graph - st.write("Spend per Key - Top 10:") + st.write("Spend ($) per Key:") top_10_df = spend_df.nlargest(10, "spend") fig = px.bar( top_10_df, x="token", y="spend", title="Top 10 Spend per Key", - height=500, # Adjust the height - width=800, # Adjust the width) + height=550, # Adjust the height + width=1200, # Adjust the width) + hover_data=["token", "spend", "user_id", "team_id"], ) st.plotly_chart(fig) @@ -400,12 +401,12 @@ def admin_page(is_admin="NOT_GIVEN", input_api_url=None, input_proxy_key=None): "Go to", ( "Connect to Proxy", - "Update Config", - "Add Models", - "List Models", - "Create Key", "View Spend Per Key", "View Spend Per User", + "List Models", + "Update Config", + "Add Models", + "Create Key", "End-User Auth", ), ) From 6a7126af9311aaac951b1b12f603ad2f3779ee81 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 23 Jan 2024 16:24:13 -0800 Subject: [PATCH 5/7] (fix) UI --- ui/admin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ui/admin.py b/ui/admin.py index 674bba7fe0d..8b5c6b3ab4e 100644 --- a/ui/admin.py +++ b/ui/admin.py @@ -210,7 +210,7 @@ def spend_per_key(): spend_df = pd.DataFrame(spend_per_key) # Display the spend per key as a graph - st.write("Spend ($) per Key:") + st.header("Spend ($) per API Key:") top_10_df = spend_df.nlargest(10, "spend") fig = px.bar( top_10_df, From 8ae8edfdb486af77af3371ab93c50f4ae2429ab2 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 23 Jan 2024 16:27:25 -0800 Subject: [PATCH 6/7] (fix) add doc string for /spend/keys --- litellm/proxy/proxy_server.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a69e379583c..cf2463226bd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2311,6 +2311,15 @@ async def info_key_fn( dependencies=[Depends(user_api_key_auth)], ) async def spend_key_fn(): + """ + View all keys created, ordered by spend + + Example Request: + ``` + curl -X GET "http://0.0.0.0:8000/spend/keys" \ +-H "Authorization: Bearer sk-1234" + ``` + """ global prisma_client try: if prisma_client is None: From 0e9339b39096adca75a9db7ab24468724753b68a Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 23 Jan 2024 16:57:51 -0800 Subject: [PATCH 7/7] (feat) /spend/logs --- litellm/proxy/proxy_server.py | 55 +++++++++++++++++++++++++++++++++++ litellm/proxy/utils.py | 20 ++++++++++++- 2 files changed, 74 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index cf2463226bd..a23d1b5f138 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -2338,6 +2338,61 @@ async def spend_key_fn(): ) +@router.get( + "/spend/logs", + tags=["Budget & Spend Tracking"], + dependencies=[Depends(user_api_key_auth)], +) +async def view_spend_logs( + request_id: Optional[str] = fastapi.Query( + default=None, + description="request_id to get spend logs for specific request_id. If none passed then pass spend logs for all requests", + ), +): + """ + View all spend logs, if request_id is provided, only logs for that request_id will be returned + + Example Request for all logs + ``` + curl -X GET "http://0.0.0.0:8000/spend/logs" \ +-H "Authorization: Bearer sk-1234" + ``` + + Example Request for specific request_id + ``` + curl -X GET "http://0.0.0.0:8000/spend/logs?request_id=chatcmpl-6dcb2540-d3d7-4e49-bb27-291f863f112e" \ +-H "Authorization: Bearer sk-1234" + ``` + """ + global prisma_client + try: + if prisma_client is None: + raise Exception( + f"Database not connected. Connect a database to your proxy - https://docs.litellm.ai/docs/simple_proxy#managing-auth---virtual-keys" + ) + spend_logs = [] + if request_id is not None: + spend_log = await prisma_client.get_data( + table_name="spend", + query_type="find_unique", + request_id=request_id, + ) + return [spend_log] + else: + spend_logs = await prisma_client.get_data( + table_name="spend", query_type="find_all" + ) + return spend_logs + + return None + + except Exception as e: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail={"error": str(e)}, + ) + + #### USER MANAGEMENT #### @router.post( "/user/new", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 2a5495919de..aecb6978bc2 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -361,7 +361,8 @@ class PrismaClient: self, token: Optional[str] = None, user_id: Optional[str] = None, - table_name: Optional[Literal["user", "key", "config"]] = None, + request_id: Optional[str] = None, + table_name: Optional[Literal["user", "key", "config", "spend"]] = None, query_type: Literal["find_unique", "find_all"] = "find_unique", ): try: @@ -411,6 +412,23 @@ class PrismaClient: } ) return response + elif table_name == "spend": + verbose_proxy_logger.debug( + f"PrismaClient: get_data: table_name == 'spend'" + ) + if request_id is not None: + response = await self.db.litellm_spendlogs.find_unique( # type: ignore + where={ + "request_id": request_id, + } + ) + return response + else: + response = await self.db.litellm_spendlogs.find_many( # type: ignore + order={"startTime": "desc"}, + ) + return response + except Exception as e: print_verbose(f"LiteLLM Prisma Client Exception: {e}") import traceback