From 2b3cc694af083ebeee4099f7218addd46b125756 Mon Sep 17 00:00:00 2001 From: ishaan-jaff Date: Mon, 15 Jan 2024 17:16:56 -0800 Subject: [PATCH] (docs) proxy - fail requests with custom error message --- docs/my-website/docs/proxy/rules.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/rules.md b/docs/my-website/docs/proxy/rules.md index 1e963577fb6..bbdb9b8a44f 100644 --- a/docs/my-website/docs/proxy/rules.md +++ b/docs/my-website/docs/proxy/rules.md @@ -8,9 +8,12 @@ Use this to fail a request based on the output of an llm api call. ```python def my_custom_rule(input): # receives the model response - if len(input) < 5: # trigger fallback if the model response is too short - return False - return True + if len(input) < 5: + return { + "decision": False, + "message": "This violates LiteLLM Proxy Rules. Response too short" + } + return {"decision": True} # message not required since, request will pass ``` ### Step 2. Point it to your proxy @@ -18,7 +21,6 @@ def my_custom_rule(input): # receives the model response ```python litellm_settings: post_call_rules: post_call_rules.my_custom_rule - num_retries: 3 ``` ### Step 3. Start + test your proxy @@ -32,7 +34,7 @@ curl --location 'http://0.0.0.0:8000/v1/chat/completions' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer sk-1234' \ --data '{ - "model": "deepseek-coder", + "model": "gpt-3.5-turbo", "messages": [{"role":"user","content":"What llm are you?"}], "temperature": 0.7, "max_tokens": 10,