From 1a33c407137f0c07558f8f039a4998b3d2da66f0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 08:38:23 -0700 Subject: [PATCH 1/4] add endpoint to disable logging for a team --- .../team_callback_endpoints.py | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 9c2ac65cc89..d51ca9ea1a2 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -190,6 +190,91 @@ async def add_team_callbacks( ) +@router.post( + "/team/{team_id}/disable_logging", + tags=["team management"], + dependencies=[Depends(user_api_key_auth)], +) +@management_endpoint_wrapper +async def disable_team_logging( + http_request: Request, + team_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + try: + from litellm.proxy.proxy_server import prisma_client + + if prisma_client is None: + raise HTTPException(status_code=500, detail={"error": "No db connected"}) + + # Check if team exists + _existing_team = await prisma_client.get_data( + team_id=team_id, table_name="team", query_type="find_unique" + ) + if _existing_team is None: + raise HTTPException( + status_code=404, + detail={"error": f"Team id = {team_id} does not exist."}, + ) + + # Update team metadata to disable logging + team_metadata = _existing_team.metadata + team_callback_settings = team_metadata.get("callback_settings", {}) + team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) + + # Reset callbacks + team_callback_settings_obj.success_callback = [] + team_callback_settings_obj.failure_callback = [] + + # Update metadata + team_metadata["callback_settings"] = team_callback_settings_obj.model_dump() + team_metadata_json = json.dumps(team_metadata) + + # Update team in database + updated_team = await prisma_client.db.litellm_teamtable.update( + where={"team_id": team_id}, data={"metadata": team_metadata_json} # type: ignore + ) + + if updated_team is None: + raise HTTPException( + status_code=404, + detail={ + "error": f"Team id = {team_id} does not exist. Error updating team logging" + }, + ) + + return { + "status": "success", + "message": f"Logging disabled for team {team_id}", + "data": { + "team_id": updated_team.team_id, + "success_callbacks": [], + "failure_callbacks": [], + }, + } + + except Exception as e: + verbose_proxy_logger.error( + f"litellm.proxy.proxy_server.disable_team_logging(): Exception occurred - {str(e)}" + ) + verbose_proxy_logger.debug(traceback.format_exc()) + if isinstance(e, HTTPException): + raise ProxyException( + message=getattr(e, "detail", f"Internal Server Error({str(e)})"), + type=ProxyErrorTypes.internal_server_error.value, + param=getattr(e, "param", "None"), + code=getattr(e, "status_code", status.HTTP_500_INTERNAL_SERVER_ERROR), + ) + elif isinstance(e, ProxyException): + raise e + raise ProxyException( + message="Internal Server Error, " + str(e), + type=ProxyErrorTypes.internal_server_error.value, + param=getattr(e, "param", "None"), + code=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + @router.get( "/team/{team_id:path}/callback", tags=["team management"], From 24ae0119d1e7e60b5b65a69e33524ef6ceae0b6d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 08:41:05 -0700 Subject: [PATCH 2/4] add debug logging for team callback settings --- litellm/proxy/litellm_pre_call_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 642c1261696..8909b1da3d0 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -213,6 +213,9 @@ async def add_litellm_data_to_request( if "callback_settings" in team_metadata: callback_settings = team_metadata.get("callback_settings", None) or {} callback_settings_obj = TeamCallbackMetadata(**callback_settings) + verbose_proxy_logger.debug( + "Team callback settings activated: %s", callback_settings_obj + ) """ callback_settings = { { From 69091f31dfd412de274bd713879962779e40638e Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 08:43:01 -0700 Subject: [PATCH 3/4] feat - add success_Callback per request --- litellm/proxy/proxy_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 2508a48a1df..60ddfba32ba 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -12,4 +12,4 @@ general_settings: master_key: sk-1234 litellm_settings: - callbacks: ["arize"] \ No newline at end of file + success_callback: ["langfuse"] \ No newline at end of file From 25dc0877d9316588f48629324c9d73fbae2c47b3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 23 Jul 2024 09:16:53 -0700 Subject: [PATCH 4/4] docs Debugging / Troubleshooting --- docs/my-website/docs/proxy/team_logging.md | 70 ++++++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index c3758a7c713..3e6594efccb 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -10,12 +10,14 @@ Allow each team to use their own Langfuse Project / custom callbacks ``` Team 1 -> Logs to Langfuse Project 1 Team 2 -> Logs to Langfuse Project 2 -Team 3 -> Logs to Langsmith +Team 3 -> Disabled Logging (for GDPR compliance) ``` -## Quick Start +## Set Callbacks Per Team -## 1. Set callback for team +### 1. Set callback for team + +We make a request to `POST /team/{team_id}/callback` to add a callback for ```shell curl -X POST 'http:/localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/callback' \ @@ -44,7 +46,7 @@ curl -X POST 'http:/localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/cal |     `langfuse_secret_key` | string | Required | |     `langfuse_host` | string | Optional (defaults to https://cloud.langfuse.com) | -## 2. Create key for team +### 2. Create key for team All keys created for team `dbe2f686-a686-4896-864a-4c3924458709` will log to langfuse project specified on [Step 1. Set callback for team](#1-set-callback-for-team) @@ -59,7 +61,7 @@ curl --location 'http://0.0.0.0:4000/key/generate' \ ``` -## 3. Make `/chat/completion` request for team +### 3. Make `/chat/completion` request for team ```shell curl -i http://localhost:4000/v1/chat/completions \ @@ -75,6 +77,64 @@ curl -i http://localhost:4000/v1/chat/completions \ Expect this to be logged on the langfuse project specified on [Step 1. Set callback for team](#1-set-callback-for-team) + +## Disable Logging for a Team + +To disable logging for a specific team, you can use the following endpoint: + +`POST /team/{team_id}/disable_logging` + +This endpoint removes all success and failure callbacks for the specified team, effectively disabling logging. + +### Step 1. Disable logging for team + +```shell +curl -X POST 'http://localhost:4000/team/YOUR_TEAM_ID/disable_logging' \ + -H 'Authorization: Bearer YOUR_API_KEY' +``` +Replace YOUR_TEAM_ID with the actual team ID + +**Response** +A successful request will return a response similar to this: +```json +{ + "status": "success", + "message": "Logging disabled for team YOUR_TEAM_ID", + "data": { + "team_id": "YOUR_TEAM_ID", + "success_callbacks": [], + "failure_callbacks": [] + } +} +``` + +### Step 2. Test it - `/chat/completions` + +Use a key generated for team = `team_id` - you should see no logs on your configured success callback (eg. Langfuse) + +```shell +curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-KbUuE0WNptC0jXapyMmLBA" \ + -d '{ + "model": "gpt-4", + "messages": [ + {"role": "user", "content": "Hello, Claude gm!"} + ] +}' +``` + +### Debugging / Troubleshooting + +- Check active callbacks for team using `GET /team/{team_id}/callback` + +Use this to check what success/failure callbacks are active for team=`team_id` + +```shell +curl -X GET 'http://localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/callback' \ + -H 'Authorization: Bearer sk-1234' +``` + ## Team Logging Endpoints - [`POST /team/{team_id}/callback` Add a success/failure callback to a team](https://litellm-api.up.railway.app/#/team%20management/add_team_callbacks_team__team_id__callback_post)