From 66d552eb9a53dc39afc23d9bb316b060fbedf529 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 14:51:23 -0800 Subject: [PATCH 1/8] (feat) /daily/metrics --- litellm/integrations/clickhouse.py | 120 +++++++++++++++++++++++++++++ litellm/proxy/proxy_server.py | 56 ++++++++++++++ 2 files changed, 176 insertions(+) diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 82ecd3256d7..70042366445 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -27,6 +27,126 @@ import litellm, uuid from litellm._logging import print_verbose, verbose_logger +def create_client(): + try: + import clickhouse_connect + + port = os.getenv("CLICKHOUSE_PORT") + clickhouse_host = os.getenv("CLICKHOUSE_HOST") + if clickhouse_host is not None: + verbose_logger.debug("setting up clickhouse") + if port is not None and isinstance(port, str): + port = int(port) + + client = clickhouse_connect.get_client( + host=os.getenv("CLICKHOUSE_HOST"), + port=port, + username=os.getenv("CLICKHOUSE_USERNAME"), + password=os.getenv("CLICKHOUSE_PASSWORD"), + ) + return client + else: + raise Exception("Clickhouse: Clickhouse host not set") + except Exception as e: + raise ValueError(f"Clickhouse: {e}") + + +def build_daily_metrics(): + click_house_client = create_client() + + # get daily spend + daily_spend = click_house_client.query_df( + """ + SELECT sumMerge(DailySpend) as daily_spend, day FROM daily_aggregated_spend GROUP BY day + """ + ) + + # get daily spend per model + + daily_spend_per_model = click_house_client.query_df( + """ + SELECT sumMerge(DailySpend) as daily_spend, day, model FROM daily_aggregated_spend_per_model GROUP BY day, model + """ + ) + new_df = daily_spend_per_model.to_dict(orient="records") + import pandas as pd + + df = pd.DataFrame(new_df) + # Group by 'day' and create a dictionary for each group + result_dict = {} + for day, group in df.groupby("day"): + models = group["model"].tolist() + spend = group["daily_spend"].tolist() + spend_per_model = {model: spend for model, spend in zip(models, spend)} + result_dict[day] = spend_per_model + + # Display the resulting dictionary + + daily_spend_per_api_key = click_house_client.query_df( + """ + SELECT + daily_spend, + day, + api_key + FROM ( + SELECT + sumMerge(DailySpend) as daily_spend, + day, + api_key, + RANK() OVER (PARTITION BY day ORDER BY sumMerge(DailySpend) DESC) as spend_rank + FROM + daily_aggregated_spend_per_api_key + GROUP BY + day, + api_key + ) AS ranked_api_keys + WHERE + spend_rank <= 5 + AND day IS NOT NULL + ORDER BY + day, + daily_spend DESC + """ + ) + new_df = daily_spend_per_api_key.to_dict(orient="records") + import pandas as pd + + df = pd.DataFrame(new_df) + # Group by 'day' and create a dictionary for each group + api_key_result_dict = {} + for day, group in df.groupby("day"): + api_keys = group["api_key"].tolist() + spend = group["daily_spend"].tolist() + spend_per_api_key = {api_key: spend for api_key, spend in zip(api_keys, spend)} + api_key_result_dict[day] = spend_per_api_key + + # Display the resulting dictionary + + # for each day in daily spend, look up the day in result_dict and api_key_result_dict + # Assuming daily_spend DataFrame has 'day' column + result = [] + for index, row in daily_spend.iterrows(): + day = row["day"] + data_day = row.to_dict() + + # Look up in result_dict + if day in result_dict: + spend_per_model = result_dict[day] + # Assuming there is a column named 'model' in daily_spend + data_day["spend_per_model"] = spend_per_model # Assign 0 if model not found + + # Look up in api_key_result_dict + if day in api_key_result_dict: + spend_per_api_key = api_key_result_dict[day] + # Assuming there is a column named 'api_key' in daily_spend + data_day["spend_per_api_key"] = spend_per_api_key + + result.append(data_day) + + # print("FINAL daily metric", result) + return result + + def _create_clickhouse_material_views(client=None, table_names=[]): # Create Materialized Views if they don't exist # Materialized Views send new inserted rows to the aggregate tables diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 6b9e3f644c4..d4b261817da 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3991,6 +3991,62 @@ async def view_spend_logs( ) +@router.get( + "/daily_metrics", + summary="Get daily spend metrics", + tags=["budget & spend Tracking"], + dependencies=[Depends(user_api_key_auth)], +) +async def view_daily_metrics( + start_date: Optional[str] = fastapi.Query( + default=None, + description="Time from which to start viewing key spend", + ), + end_date: Optional[str] = fastapi.Query( + default=None, + description="Time till which to view key spend", + ), + metric_type: Optional[Literal["per_api_key", "per_user", "per_model"]] = None, +): + """ """ + try: + if os.getenv("CLICKHOUSE_HOST") is not None: + # gettting spend logs from clickhouse + from litellm.integrations import clickhouse + + return clickhouse.build_daily_metrics() + + # create a response object + """ + { + "date": "2022-01-01", + "spend": 0.0, + "users": {}, + "models": {} + } + """ + else: + raise Exception( + "Clickhouse: Clickhouse host not set. Required for viewing /daily/metrics" + ) + except Exception as e: + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"/spend/logs Error({str(e)})"), + type="internal_error", + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="/spend/logs Error" + str(e), + type="internal_error", + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + #### USER MANAGEMENT #### @router.post( "/user/new", From 7cabc6ac565c0b03392daedd2a81d2f4482396e9 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 18:03:15 -0800 Subject: [PATCH 2/8] (docs) show daily metrics tab --- docs/my-website/docs/proxy/metrics.md | 32 +++++++++++++++++++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 33 insertions(+) create mode 100644 docs/my-website/docs/proxy/metrics.md diff --git a/docs/my-website/docs/proxy/metrics.md b/docs/my-website/docs/proxy/metrics.md new file mode 100644 index 00000000000..d113ff6fb70 --- /dev/null +++ b/docs/my-website/docs/proxy/metrics.md @@ -0,0 +1,32 @@ +# 💸 GET Daily Spend, Usage Metrics + +## Response format +```json +[ + { + "daily_spend": 7.9261938052047e+16, + "day": "2024-02-01T00:00:00", + "spend_per_model": {"azure/gpt-4": 7.9261938052047e+16}, + "spend_per_api_key": { + "76": 914495704992000.0, + "12": 905726697912000.0, + "71": 866312628003000.0, + "28": 865461799332000.0, + "13": 859151538396000.0 + } + }, + { + "daily_spend": 7.938489251309491e+16, + "day": "2024-02-02T00:00:00", + "spend_per_model": {"gpt-3.5": 7.938489251309491e+16}, + "spend_per_api_key": { + "91": 896805036036000.0, + "78": 889692646082000.0, + "49": 885386687861000.0, + "28": 873869890984000.0, + "56": 867398637692000.0 + } + } +] + +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 7a24723af6f..d69abcbfbde 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -40,6 +40,7 @@ const sidebars = { "proxy/virtual_keys", "proxy/users", "proxy/ui", + "proxy/metrics", "proxy/model_management", "proxy/health", "proxy/debugging", From d19370083d711aa607b15284aaac70d06dfbab70 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 18:33:59 -0800 Subject: [PATCH 3/8] (feat) update /daily metrics --- docs/my-website/docs/proxy/metrics.md | 53 +++++++++++++++------------ litellm/integrations/clickhouse.py | 31 ++++++++++++++-- 2 files changed, 58 insertions(+), 26 deletions(-) diff --git a/docs/my-website/docs/proxy/metrics.md b/docs/my-website/docs/proxy/metrics.md index d113ff6fb70..5b839dbd44f 100644 --- a/docs/my-website/docs/proxy/metrics.md +++ b/docs/my-website/docs/proxy/metrics.md @@ -3,30 +3,37 @@ ## Response format ```json [ - { - "daily_spend": 7.9261938052047e+16, - "day": "2024-02-01T00:00:00", - "spend_per_model": {"azure/gpt-4": 7.9261938052047e+16}, - "spend_per_api_key": { - "76": 914495704992000.0, - "12": 905726697912000.0, - "71": 866312628003000.0, - "28": 865461799332000.0, - "13": 859151538396000.0 + daily_spend = [ + { + "daily_spend": 7.9261938052047e+16, + "day": "2024-02-01T00:00:00", + "spend_per_model": {"azure/gpt-4": 7.9261938052047e+16}, + "spend_per_api_key": { + "76": 914495704992000.0, + "12": 905726697912000.0, + "71": 866312628003000.0, + "28": 865461799332000.0, + "13": 859151538396000.0 + } + }, + { + "daily_spend": 7.938489251309491e+16, + "day": "2024-02-02T00:00:00", + "spend_per_model": {"gpt-3.5": 7.938489251309491e+16}, + "spend_per_api_key": { + "91": 896805036036000.0, + "78": 889692646082000.0, + "49": 885386687861000.0, + "28": 873869890984000.0, + "56": 867398637692000.0 + } } - }, - { - "daily_spend": 7.938489251309491e+16, - "day": "2024-02-02T00:00:00", - "spend_per_model": {"gpt-3.5": 7.938489251309491e+16}, - "spend_per_api_key": { - "91": 896805036036000.0, - "78": 889692646082000.0, - "49": 885386687861000.0, - "28": 873869890984000.0, - "56": 867398637692000.0 - } - } + + ], + total_spend = 200, + top_models = {"gpt4": 0.2, "vertexai/gemini-pro":10}, + top_api_keys = {"899922": 0.9, "838hcjd999seerr88": 20} + ] ``` \ No newline at end of file diff --git a/litellm/integrations/clickhouse.py b/litellm/integrations/clickhouse.py index 70042366445..55eaaa090a4 100644 --- a/litellm/integrations/clickhouse.py +++ b/litellm/integrations/clickhouse.py @@ -62,7 +62,6 @@ def build_daily_metrics(): ) # get daily spend per model - daily_spend_per_model = click_house_client.query_df( """ SELECT sumMerge(DailySpend) as daily_spend, day, model FROM daily_aggregated_spend_per_model GROUP BY day, model @@ -82,6 +81,7 @@ def build_daily_metrics(): # Display the resulting dictionary + # get daily spend per API key daily_spend_per_api_key = click_house_client.query_df( """ SELECT @@ -122,6 +122,23 @@ def build_daily_metrics(): # Display the resulting dictionary + # Calculate total spend across all days + total_spend = daily_spend["daily_spend"].sum() + + # Identify top models and top API keys with the highest spend across all days + top_models = {} + top_api_keys = {} + + for day, spend_per_model in result_dict.items(): + for model, model_spend in spend_per_model.items(): + if model not in top_models or model_spend > top_models[model]: + top_models[model] = model_spend + + for day, spend_per_api_key in api_key_result_dict.items(): + for api_key, api_key_spend in spend_per_api_key.items(): + if api_key not in top_api_keys or api_key_spend > top_api_keys[api_key]: + top_api_keys[api_key] = api_key_spend + # for each day in daily spend, look up the day in result_dict and api_key_result_dict # Assuming daily_spend DataFrame has 'day' column result = [] @@ -143,8 +160,16 @@ def build_daily_metrics(): result.append(data_day) - # print("FINAL daily metric", result) - return result + data_to_return = {} + data_to_return["daily_spend"] = result + + data_to_return["total_spend"] = total_spend + data_to_return["top_models"] = top_models + data_to_return["top_api_keys"] = top_api_keys + return data_to_return + + +# build_daily_metrics() def _create_clickhouse_material_views(client=None, table_names=[]): From 8c5cdb6744eb5a1bfede9d0738fdea3580f78269 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 18:36:16 -0800 Subject: [PATCH 4/8] (feat) update /daily metrics --- litellm/proxy/proxy_server.py | 49 ++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d4b261817da..b00e7f231ee 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3832,13 +3832,51 @@ async def view_spend_logs( # gettting spend logs from clickhouse from litellm.proxy.enterprise.utils import view_spend_logs_from_clickhouse - return await view_spend_logs_from_clickhouse( - api_key=api_key, - user_id=user_id, - request_id=request_id, + daily_metrics = await view_daily_metrics( start_date=start_date, end_date=end_date, ) + + # get the top api keys across all daily_metrics + top_api_keys = {} # type: ignore + + # make this compatible with the admin UI + for response in daily_metrics.get("daily_spend", {}): + response["startTime"] = response["day"] + response["spend"] = response["daily_spend"] + response["models"] = response["spend_per_model"] + response["users"] = {"ishaan": 0.0} + spend_per_api_key = response["spend_per_api_key"] + + # insert spend_per_api_key key, values in response + for key, value in spend_per_api_key.items(): + response[key] = value + top_api_keys[key] = top_api_keys.get(key, 0.0) + value + + del response["day"] + del response["daily_spend"] + del response["spend_per_model"] + del response["spend_per_api_key"] + + # get top 5 api keys + top_api_keys = sorted(top_api_keys.items(), key=lambda x: x[1], reverse=True) # type: ignore + top_api_keys = top_api_keys[:5] # type: ignore + top_api_keys = dict(top_api_keys) # type: ignore + """ + set it like this + { + "key" : key, + "spend:" : spend + } + """ + response_keys = {} + for key in top_api_keys.items(): + response_keys["key"] = key[0] + response_keys["spend"] = key[1] + + daily_metrics["top_api_keys"] = response_keys + + return daily_metrics global prisma_client try: verbose_proxy_logger.debug("inside view_spend_logs") @@ -4006,7 +4044,6 @@ async def view_daily_metrics( default=None, description="Time till which to view key spend", ), - metric_type: Optional[Literal["per_api_key", "per_user", "per_model"]] = None, ): """ """ try: @@ -4022,7 +4059,7 @@ async def view_daily_metrics( "date": "2022-01-01", "spend": 0.0, "users": {}, - "models": {} + "models": {}, } """ else: From 4fb723f119b5836e96ceea1f550eb618d271faf3 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:34:47 -0800 Subject: [PATCH 5/8] (docs) show /daily_metrics --- docs/my-website/docs/proxy/metrics.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/my-website/docs/proxy/metrics.md b/docs/my-website/docs/proxy/metrics.md index 5b839dbd44f..bf5ebe2858e 100644 --- a/docs/my-website/docs/proxy/metrics.md +++ b/docs/my-website/docs/proxy/metrics.md @@ -1,5 +1,10 @@ # 💸 GET Daily Spend, Usage Metrics +## Request Format +```shell +curl -X GET "http://0.0.0.0:4000/daily_metrics" -H "Authorization: Bearer sk-1234" +``` + ## Response format ```json [ From e6d13261ce83aae1ac09cb619a3e7ed2204c597f Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:35:18 -0800 Subject: [PATCH 6/8] (feat) show analytics on admin UI --- litellm/proxy/proxy_server.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b00e7f231ee..f174f668565 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -3869,11 +3869,15 @@ async def view_spend_logs( "spend:" : spend } """ - response_keys = {} + # we need this to show on the Admin UI + response_keys = [] for key in top_api_keys.items(): - response_keys["key"] = key[0] - response_keys["spend"] = key[1] - + response_keys.append( + { + "key": key[0], + "spend": key[1], + } + ) daily_metrics["top_api_keys"] = response_keys return daily_metrics From 02dc031bb599896df91324ba60941907d5f2ae95 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:35:40 -0800 Subject: [PATCH 7/8] (ui) show usage on ui --- ui/litellm-dashboard/src/components/usage.tsx | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/ui/litellm-dashboard/src/components/usage.tsx b/ui/litellm-dashboard/src/components/usage.tsx index 84c942eec6e..53493eabce1 100644 --- a/ui/litellm-dashboard/src/components/usage.tsx +++ b/ui/litellm-dashboard/src/components/usage.tsx @@ -172,20 +172,32 @@ const UsagePage: React.FC = ({ startTime, endTime ).then(async (response) => { - const topKeysResponse = await keyInfoCall( - accessToken, - getTopKeys(response) - ); - const filtered_keys = topKeysResponse["info"].map((k: any) => ({ - key: (k["key_name"] || k["key_alias"] || k["token"]).substring( - 0, - 7 - ), - spend: k["spend"], - })); - setTopKeys(filtered_keys); - setTopUsers(getTopUsers(response)); - setKeySpendData(response); + console.log("result from spend logs call", response); + if ("daily_spend" in response) { + // this is from clickhouse analytics + // + let daily_spend = response["daily_spend"]; + console.log("daily spend", daily_spend); + setKeySpendData(daily_spend); + let topApiKeys = response.top_api_keys; + setTopKeys(topApiKeys); + } + else { + const topKeysResponse = await keyInfoCall( + accessToken, + getTopKeys(response) + ); + const filtered_keys = topKeysResponse["info"].map((k: any) => ({ + key: (k["key_name"] || k["key_alias"] || k["token"]).substring( + 0, + 7 + ), + spend: k["spend"], + })); + setTopKeys(filtered_keys); + setTopUsers(getTopUsers(response)); + setKeySpendData(response); + } }); } catch (error) { console.error("There was an error fetching the data", error); From d9a3c4179f50020cfc60b18109c0e62c395a6456 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Tue, 27 Feb 2024 19:38:18 -0800 Subject: [PATCH 8/8] (feat) pin pandas on req.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 6bd69302fa0..caede5b67f6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ gunicorn==21.2.0 # server dep boto3==1.34.34 # aws bedrock/sagemaker calls redis==5.0.0 # caching numpy==1.24.3 # semantic caching +pandas==2.1.1 # for viewing clickhouse spend analytics prisma==0.11.0 # for db mangum==0.17.0 # for aws lambda functions google-generativeai==0.3.2 # for vertex ai calls