From bbdcc75c601b4a15f78ce39d0b3933754e52d803 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:33:06 -0700 Subject: [PATCH 1/3] fix log failures for key based logging --- litellm/proxy/litellm_pre_call_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index d41aae50f6e..890c576c944 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -86,10 +86,11 @@ def convert_key_logging_metadata_to_callback( team_callback_settings_obj.success_callback = [] if team_callback_settings_obj.failure_callback is None: team_callback_settings_obj.failure_callback = [] + if data.callback_name not in team_callback_settings_obj.success_callback: team_callback_settings_obj.success_callback.append(data.callback_name) - if data.callback_name in team_callback_settings_obj.failure_callback: + if data.callback_name not in team_callback_settings_obj.failure_callback: team_callback_settings_obj.failure_callback.append(data.callback_name) for var, value in data.callback_vars.items(): From 57ebe4649eeec5ce8348dc383f8ba31094415546 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:44:37 -0700 Subject: [PATCH 2/3] add test for using success and failure --- litellm/tests/test_proxy_server.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/litellm/tests/test_proxy_server.py b/litellm/tests/test_proxy_server.py index fb1025ab26b..102c126d110 100644 --- a/litellm/tests/test_proxy_server.py +++ b/litellm/tests/test_proxy_server.py @@ -1255,7 +1255,17 @@ async def test_add_callback_via_key(prisma_client): @pytest.mark.asyncio -async def test_add_callback_via_key_litellm_pre_call_utils(prisma_client): +@pytest.mark.parametrize( + "callback_type, expected_success_callbacks, expected_failure_callbacks", + [ + ("success", ["langfuse"], []), + ("failure", [], ["langfuse"]), + ("success_and_failure", ["langfuse"], ["langfuse"]), + ], +) +async def test_add_callback_via_key_litellm_pre_call_utils( + prisma_client, callback_type, expected_success_callbacks, expected_failure_callbacks +): import json from fastapi import HTTPException, Request, Response @@ -1312,7 +1322,7 @@ async def test_add_callback_via_key_litellm_pre_call_utils(prisma_client): "logging": [ { "callback_name": "langfuse", - "callback_type": "success", + "callback_type": callback_type, "callback_vars": { "langfuse_public_key": "my-mock-public-key", "langfuse_secret_key": "my-mock-secret-key", @@ -1359,14 +1369,21 @@ async def test_add_callback_via_key_litellm_pre_call_utils(prisma_client): } new_data = await add_litellm_data_to_request(**data) + print("NEW DATA: {}".format(new_data)) - assert "success_callback" in new_data - assert new_data["success_callback"] == ["langfuse"] assert "langfuse_public_key" in new_data assert new_data["langfuse_public_key"] == "my-mock-public-key" assert "langfuse_secret_key" in new_data assert new_data["langfuse_secret_key"] == "my-mock-secret-key" + if expected_success_callbacks: + assert "success_callback" in new_data + assert new_data["success_callback"] == expected_success_callbacks + + if expected_failure_callbacks: + assert "failure_callback" in new_data + assert new_data["failure_callback"] == expected_failure_callbacks + @pytest.mark.asyncio async def test_gemini_pass_through_endpoint(): From 949af7be2ec114b9d9bae792f504ffc68ceb8417 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 9 Sep 2024 16:49:26 -0700 Subject: [PATCH 3/3] fix team based logging doc --- docs/my-website/docs/proxy/team_logging.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index c593f23bf5a..fb177da761f 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -208,8 +208,8 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ -d '{ "metadata": { "logging": [{ - "callback_name": "langfuse", # 'otel', 'langfuse', 'lunary' - "callback_type": "success" # set, if required by integration - future improvement, have logging tools work for success + failure by default + "callback_name": "langfuse", # "otel", "langfuse", "lunary" + "callback_type": "success", # "success", "failure", "success_and_failure" "callback_vars": { "langfuse_public_key": "os.environ/LANGFUSE_PUBLIC_KEY", # [RECOMMENDED] reference key in proxy environment "langfuse_secret_key": "os.environ/LANGFUSE_SECRET_KEY", # [RECOMMENDED] reference key in proxy environment