Merge pull request #2226 from BerriAI/litellm_daily_metrics

[FEAT] GET /daily_metrics
2026-10-09 03:18:44 +00:00 · 2024-02-27 20:33:58 -08:00 · 2024-02-27 20:33:58 -08:00 · 3eb2962580
commit 3eb2962580
parent 551669ad61 990439c49c
6 changed files with 318 additions and 18 deletions
--- a/docs/my-website/docs/proxy/metrics.md
+++ b/docs/my-website/docs/proxy/metrics.md
@ -0,0 +1,44 @@
+# 💸 GET Daily Spend, Usage Metrics
+
+## Request Format
+```shell
+curl -X GET "http://0.0.0.0:4000/daily_metrics" -H "Authorization: Bearer sk-1234"
+```
+
+## Response format 
+```json
+[
+    daily_spend = [
+        {
+            "daily_spend": 7.9261938052047e+16,
+            "day": "2024-02-01T00:00:00",
+            "spend_per_model": {"azure/gpt-4": 7.9261938052047e+16},
+            "spend_per_api_key": {
+                "76": 914495704992000.0,
+                "12": 905726697912000.0,
+                "71": 866312628003000.0,
+                "28": 865461799332000.0,
+                "13": 859151538396000.0
+            }
+        },
+        {
+            "daily_spend": 7.938489251309491e+16,
+            "day": "2024-02-02T00:00:00",
+            "spend_per_model": {"gpt-3.5": 7.938489251309491e+16},
+            "spend_per_api_key": {
+                "91": 896805036036000.0,
+                "78": 889692646082000.0,
+                "49": 885386687861000.0,
+                "28": 873869890984000.0,
+                "56": 867398637692000.0
+            }
+        }
+
+    ],
+    total_spend = 200,
+    top_models = {"gpt4": 0.2, "vertexai/gemini-pro":10},
+    top_api_keys = {"899922": 0.9, "838hcjd999seerr88": 20}
+
+]
+
+```
--- a/docs/my-website/sidebars.js
+++ b/docs/my-website/sidebars.js
@ -40,6 +40,7 @@ const sidebars = {
        "proxy/virtual_keys",
        "proxy/users",
        "proxy/ui",
+        "proxy/metrics",
        "proxy/model_management",
        "proxy/health",
        "proxy/debugging",
--- a/litellm/integrations/clickhouse.py
+++ b/litellm/integrations/clickhouse.py
@ -27,6 +27,151 @@ import litellm, uuid
 from litellm._logging import print_verbose, verbose_logger


+def create_client():
+    try:
+        import clickhouse_connect
+
+        port = os.getenv("CLICKHOUSE_PORT")
+        clickhouse_host = os.getenv("CLICKHOUSE_HOST")
+        if clickhouse_host is not None:
+            verbose_logger.debug("setting up clickhouse")
+            if port is not None and isinstance(port, str):
+                port = int(port)
+
+            client = clickhouse_connect.get_client(
+                host=os.getenv("CLICKHOUSE_HOST"),
+                port=port,
+                username=os.getenv("CLICKHOUSE_USERNAME"),
+                password=os.getenv("CLICKHOUSE_PASSWORD"),
+            )
+            return client
+        else:
+            raise Exception("Clickhouse: Clickhouse host not set")
+    except Exception as e:
+        raise ValueError(f"Clickhouse: {e}")
+
+
+def build_daily_metrics():
+    click_house_client = create_client()
+
+    # get daily spend
+    daily_spend = click_house_client.query_df(
+        """
+        SELECT sumMerge(DailySpend) as daily_spend, day FROM daily_aggregated_spend GROUP BY day
+        """
+    )
+
+    # get daily spend per model
+    daily_spend_per_model = click_house_client.query_df(
+        """
+        SELECT sumMerge(DailySpend) as daily_spend, day, model FROM daily_aggregated_spend_per_model GROUP BY day, model
+        """
+    )
+    new_df = daily_spend_per_model.to_dict(orient="records")
+    import pandas as pd
+
+    df = pd.DataFrame(new_df)
+    # Group by 'day' and create a dictionary for each group
+    result_dict = {}
+    for day, group in df.groupby("day"):
+        models = group["model"].tolist()
+        spend = group["daily_spend"].tolist()
+        spend_per_model = {model: spend for model, spend in zip(models, spend)}
+        result_dict[day] = spend_per_model
+
+    # Display the resulting dictionary
+
+    # get daily spend per API key
+    daily_spend_per_api_key = click_house_client.query_df(
+        """
+            SELECT
+                daily_spend,
+                day,
+                api_key
+            FROM (
+                SELECT
+                    sumMerge(DailySpend) as daily_spend,
+                    day,
+                    api_key,
+                    RANK() OVER (PARTITION BY day ORDER BY sumMerge(DailySpend) DESC) as spend_rank
+                FROM
+                    daily_aggregated_spend_per_api_key
+                GROUP BY
+                    day,
+                    api_key
+            ) AS ranked_api_keys
+            WHERE
+                spend_rank <= 5
+                AND day IS NOT NULL
+            ORDER BY
+                day,
+                daily_spend DESC
+        """
+    )
+    new_df = daily_spend_per_api_key.to_dict(orient="records")
+    import pandas as pd
+
+    df = pd.DataFrame(new_df)
+    # Group by 'day' and create a dictionary for each group
+    api_key_result_dict = {}
+    for day, group in df.groupby("day"):
+        api_keys = group["api_key"].tolist()
+        spend = group["daily_spend"].tolist()
+        spend_per_api_key = {api_key: spend for api_key, spend in zip(api_keys, spend)}
+        api_key_result_dict[day] = spend_per_api_key
+
+    # Display the resulting dictionary
+
+    # Calculate total spend across all days
+    total_spend = daily_spend["daily_spend"].sum()
+
+    # Identify top models and top API keys with the highest spend across all days
+    top_models = {}
+    top_api_keys = {}
+
+    for day, spend_per_model in result_dict.items():
+        for model, model_spend in spend_per_model.items():
+            if model not in top_models or model_spend > top_models[model]:
+                top_models[model] = model_spend
+
+    for day, spend_per_api_key in api_key_result_dict.items():
+        for api_key, api_key_spend in spend_per_api_key.items():
+            if api_key not in top_api_keys or api_key_spend > top_api_keys[api_key]:
+                top_api_keys[api_key] = api_key_spend
+
+    # for each day in daily spend, look up the day in result_dict and api_key_result_dict
+    # Assuming daily_spend DataFrame has 'day' column
+    result = []
+    for index, row in daily_spend.iterrows():
+        day = row["day"]
+        data_day = row.to_dict()
+
+        # Look up in result_dict
+        if day in result_dict:
+            spend_per_model = result_dict[day]
+            # Assuming there is a column named 'model' in daily_spend
+            data_day["spend_per_model"] = spend_per_model  # Assign 0 if model not found
+
+        # Look up in api_key_result_dict
+        if day in api_key_result_dict:
+            spend_per_api_key = api_key_result_dict[day]
+            # Assuming there is a column named 'api_key' in daily_spend
+            data_day["spend_per_api_key"] = spend_per_api_key
+
+        result.append(data_day)
+
+    data_to_return = {}
+    data_to_return["daily_spend"] = result
+
+    data_to_return["total_spend"] = total_spend
+    data_to_return["top_models"] = top_models
+    data_to_return["top_api_keys"] = top_api_keys
+    return data_to_return
+
+
+# build_daily_metrics()
+
+
 def _start_clickhouse():
    import clickhouse_connect

--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@ -3833,13 +3833,55 @@ async def view_spend_logs(
        # gettting spend logs from clickhouse
        from litellm.proxy.enterprise.utils import view_spend_logs_from_clickhouse

-        return await view_spend_logs_from_clickhouse(
-            api_key=api_key,
-            user_id=user_id,
-            request_id=request_id,
+        daily_metrics = await view_daily_metrics(
            start_date=start_date,
            end_date=end_date,
        )
+
+        # get the top api keys across all daily_metrics
+        top_api_keys = {}  # type: ignore
+
+        # make this compatible with the admin UI
+        for response in daily_metrics.get("daily_spend", {}):
+            response["startTime"] = response["day"]
+            response["spend"] = response["daily_spend"]
+            response["models"] = response["spend_per_model"]
+            response["users"] = {"ishaan": 0.0}
+            spend_per_api_key = response["spend_per_api_key"]
+
+            # insert spend_per_api_key key, values in response
+            for key, value in spend_per_api_key.items():
+                response[key] = value
+                top_api_keys[key] = top_api_keys.get(key, 0.0) + value
+
+            del response["day"]
+            del response["daily_spend"]
+            del response["spend_per_model"]
+            del response["spend_per_api_key"]
+
+        # get top 5 api keys
+        top_api_keys = sorted(top_api_keys.items(), key=lambda x: x[1], reverse=True)  # type: ignore
+        top_api_keys = top_api_keys[:5]  # type: ignore
+        top_api_keys = dict(top_api_keys)  # type: ignore
+        """
+        set it like this 
+        {
+            "key" : key,
+            "spend:" : spend
+        }
+        """
+        # we need this to show on the Admin UI
+        response_keys = []
+        for key in top_api_keys.items():
+            response_keys.append(
+                {
+                    "key": key[0],
+                    "spend": key[1],
+                }
+            )
+        daily_metrics["top_api_keys"] = response_keys
+
+        return daily_metrics
    global prisma_client
    try:
        verbose_proxy_logger.debug("inside view_spend_logs")
@ -3992,6 +4034,61 @@ async def view_spend_logs(
        )


+@router.get(
+    "/daily_metrics",
+    summary="Get daily spend metrics",
+    tags=["budget & spend Tracking"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def view_daily_metrics(
+    start_date: Optional[str] = fastapi.Query(
+        default=None,
+        description="Time from which to start viewing key spend",
+    ),
+    end_date: Optional[str] = fastapi.Query(
+        default=None,
+        description="Time till which to view key spend",
+    ),
+):
+    """ """
+    try:
+        if os.getenv("CLICKHOUSE_HOST") is not None:
+            # gettting spend logs from clickhouse
+            from litellm.integrations import clickhouse
+
+            return clickhouse.build_daily_metrics()
+
+            # create a response object
+            """
+            {
+                "date": "2022-01-01",
+                "spend": 0.0,
+                "users": {},
+                "models": {},
+            }
+            """
+        else:
+            raise Exception(
+                "Clickhouse: Clickhouse host not set. Required for viewing /daily/metrics"
+            )
+    except Exception as e:
+        if isinstance(e, HTTPException):
+            raise ProxyException(
+                message=getattr(e, "detail", f"/spend/logs Error({str(e)})"),
+                type="internal_error",
+                param=getattr(e, "param", "None"),
+                code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR),
+            )
+        elif isinstance(e, ProxyException):
+            raise e
+        raise ProxyException(
+            message="/spend/logs Error" + str(e),
+            type="internal_error",
+            param=getattr(e, "param", "None"),
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
+
+
 #### USER MANAGEMENT ####
@router.post(
    "/user/new",
--- a/requirements.txt
+++ b/requirements.txt
@ -10,6 +10,7 @@ gunicorn==21.2.0 # server dep
 boto3==1.34.34 # aws bedrock/sagemaker calls
 redis==5.0.0 # caching
 numpy==1.24.3 # semantic caching
+pandas==2.1.1 # for viewing clickhouse spend analytics
 prisma==0.11.0 # for db
 mangum==0.17.0 # for aws lambda functions
 google-generativeai==0.3.2 # for vertex ai calls
--- a/ui/litellm-dashboard/src/components/usage.tsx
+++ b/ui/litellm-dashboard/src/components/usage.tsx
@ -172,20 +172,32 @@ const UsagePage: React.FC<UsagePageProps> = ({
            startTime,
            endTime
          ).then(async (response) => {
-            const topKeysResponse = await keyInfoCall(
-              accessToken,
-              getTopKeys(response)
-            );
-            const filtered_keys = topKeysResponse["info"].map((k: any) => ({
-              key: (k["key_name"] || k["key_alias"] || k["token"]).substring(
-                0,
-                7
-              ),
-              spend: k["spend"],
-            }));
-            setTopKeys(filtered_keys);
-            setTopUsers(getTopUsers(response));
-            setKeySpendData(response);
+            console.log("result from spend logs call", response);
+            if ("daily_spend" in response) {
+              // this is from clickhouse analytics
+              // 
+              let daily_spend = response["daily_spend"];
+              console.log("daily spend", daily_spend);
+              setKeySpendData(daily_spend);
+              let topApiKeys = response.top_api_keys;
+              setTopKeys(topApiKeys);
+            }
+            else {
+              const topKeysResponse = await keyInfoCall(
+                accessToken,
+                getTopKeys(response)
+              );
+              const filtered_keys = topKeysResponse["info"].map((k: any) => ({
+                key: (k["key_name"] || k["key_alias"] || k["token"]).substring(
+                  0,
+                  7
+                ),
+                spend: k["spend"],
+              }));
+              setTopKeys(filtered_keys);
+              setTopUsers(getTopUsers(response));
+              setKeySpendData(response);
+            }
          });
        } catch (error) {
          console.error("There was an error fetching the data", error);