From 7bdc94ac1abd8bddbf86654a491e2385e11c4be3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 3 Jun 2024 11:28:17 -0700 Subject: [PATCH] docs - Viewing RAW CURL sent from LiteLLM --- docs/my-website/docs/proxy/logging.md | 478 +++++++++++++++----------- 1 file changed, 286 insertions(+), 192 deletions(-) diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 67c27c20474..f4dee6256a9 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -7,9 +7,9 @@ import TabItem from '@theme/TabItem'; Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTelemetry, LangFuse, DynamoDB, s3 Bucket +- [Logging to Langfuse](#logging-proxy-inputoutput---langfuse) - [Async Custom Callbacks](#custom-callback-class-async) - [Async Custom Callback APIs](#custom-callback-apis-async) -- [Logging to Langfuse](#logging-proxy-inputoutput---langfuse) - [Logging to OpenMeter](#logging-proxy-inputoutput---langfuse) - [Logging to s3 Buckets](#logging-proxy-inputoutput---s3-buckets) - [Logging to DataDog](#logging-proxy-inputoutput---datadog) @@ -19,6 +19,291 @@ Log Proxy Input, Output, Exceptions using Custom Callbacks, Langfuse, OpenTeleme - [Logging to Athina](#logging-proxy-inputoutput-athina) - [(BETA) Moderation with Azure Content-Safety](#moderation-with-azure-content-safety) +## Logging Proxy Input/Output - Langfuse +We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment + +**Step 1** Install langfuse + +```shell +pip install langfuse>=2.0.0 +``` + +**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["langfuse"] +``` + +**Step 3**: Set required env variables for logging to langfuse +```shell +export LANGFUSE_PUBLIC_KEY="pk_kk" +export LANGFUSE_SECRET_KEY="sk_ss +``` + +**Step 4**: Start the proxy, make a test request + +Start proxy +```shell +litellm --config config.yaml --debug +``` + +Test Request +``` +litellm --test +``` + +Expected output on Langfuse + + + +### Logging Metadata to Langfuse + + + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "metadata": { + "generation_name": "ishaan-test-generation", + "generation_id": "gen-id22", + "trace_id": "trace-id22", + "trace_user_id": "user-id2" + } +}' +``` + + + +Set `extra_body={"metadata": { }}` to `metadata` you want to pass + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "generation_name": "ishaan-generation-openai-client", + "generation_id": "openai-client-gen-id22", + "trace_id": "openai-client-trace-id22", + "trace_user_id": "openai-client-user-id2" + } + } +) + +print(response) +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model = "gpt-3.5-turbo", + temperature=0.1, + extra_body={ + "metadata": { + "generation_name": "ishaan-generation-langchain-client", + "generation_id": "langchain-client-gen-id22", + "trace_id": "langchain-client-trace-id22", + "trace_user_id": "langchain-client-user-id2" + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + +### Team based Logging to Langfuse + +**Example:** + +This config would send langfuse logs to 2 different langfuse projects, based on the team id + +```yaml +litellm_settings: + default_team_settings: + - team_id: my-secret-project + success_callback: ["langfuse"] + langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_1 # Project 1 + langfuse_secret: os.environ/LANGFUSE_PRIVATE_KEY_1 # Project 1 + - team_id: ishaans-secret-project + success_callback: ["langfuse"] + langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_2 # Project 2 + langfuse_secret: os.environ/LANGFUSE_SECRET_2 # Project 2 +``` + +Now, when you [generate keys](./virtual_keys.md) for this team-id + +```bash +curl -X POST 'http://0.0.0.0:4000/key/generate' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{"team_id": "ishaans-secret-project"}' +``` + +All requests made with these keys will log data to their team-specific logging. + +### Redacting Messages, Response Content from Langfuse Logging + +Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to langfuse, but request metadata will still be logged. + +```yaml +model_list: + - model_name: gpt-3.5-turbo + litellm_params: + model: gpt-3.5-turbo +litellm_settings: + success_callback: ["langfuse"] + turn_off_message_logging: True +``` + +### 🔧 Debugging - Viewing RAW CURL sent from LiteLLM to provider + + + + + +Pass `metadata` as part of the request body + +```shell +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "gpt-3.5-turbo", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "metadata": { + "log_raw_request": true + } +}' +``` + + + +Set `extra_body={"metadata": {"log_raw_request": True }}` to `metadata` you want to pass + +```python +import openai +client = openai.OpenAI( + api_key="anything", + base_url="http://0.0.0.0:4000" +) + +# request sent to model set on litellm proxy, `litellm --model` +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages = [ + { + "role": "user", + "content": "this is a test request, write a short poem" + } + ], + extra_body={ + "metadata": { + "log_raw_request": True + } + } +) + +print(response) +``` + + + +```python +from langchain.chat_models import ChatOpenAI +from langchain.prompts.chat import ( + ChatPromptTemplate, + HumanMessagePromptTemplate, + SystemMessagePromptTemplate, +) +from langchain.schema import HumanMessage, SystemMessage + +chat = ChatOpenAI( + openai_api_base="http://0.0.0.0:4000", + model = "gpt-3.5-turbo", + temperature=0.1, + extra_body={ + "metadata": { + "log_raw_request": True + } + } +) + +messages = [ + SystemMessage( + content="You are a helpful assistant that im using to make a test request to." + ), + HumanMessage( + content="test from litellm. tell me why it's amazing in 1 sentence" + ), +] +response = chat(messages) + +print(response) +``` + + + + + + + ## Custom Callback Class [Async] Use this when you want to run custom callbacks in `python` @@ -402,197 +687,6 @@ litellm_settings: Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API -## Logging Proxy Input/Output - Langfuse -We will use the `--config` to set `litellm.success_callback = ["langfuse"]` this will log all successfull LLM calls to langfuse. Make sure to set `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` in your environment - -**Step 1** Install langfuse - -```shell -pip install langfuse>=2.0.0 -``` - -**Step 2**: Create a `config.yaml` file and set `litellm_settings`: `success_callback` -```yaml -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo -litellm_settings: - success_callback: ["langfuse"] -``` - -**Step 3**: Set required env variables for logging to langfuse -```shell -export LANGFUSE_PUBLIC_KEY="pk_kk" -export LANGFUSE_SECRET_KEY="sk_ss -``` - -**Step 4**: Start the proxy, make a test request - -Start proxy -```shell -litellm --config config.yaml --debug -``` - -Test Request -``` -litellm --test -``` - -Expected output on Langfuse - - - -### Logging Metadata to Langfuse - - - - - - -Pass `metadata` as part of the request body - -```shell -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "gpt-3.5-turbo", - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - "metadata": { - "generation_name": "ishaan-test-generation", - "generation_id": "gen-id22", - "trace_id": "trace-id22", - "trace_user_id": "user-id2" - } -}' -``` - - - -Set `extra_body={"metadata": { }}` to `metadata` you want to pass - -```python -import openai -client = openai.OpenAI( - api_key="anything", - base_url="http://0.0.0.0:4000" -) - -# request sent to model set on litellm proxy, `litellm --model` -response = client.chat.completions.create( - model="gpt-3.5-turbo", - messages = [ - { - "role": "user", - "content": "this is a test request, write a short poem" - } - ], - extra_body={ - "metadata": { - "generation_name": "ishaan-generation-openai-client", - "generation_id": "openai-client-gen-id22", - "trace_id": "openai-client-trace-id22", - "trace_user_id": "openai-client-user-id2" - } - } -) - -print(response) -``` - - - -```python -from langchain.chat_models import ChatOpenAI -from langchain.prompts.chat import ( - ChatPromptTemplate, - HumanMessagePromptTemplate, - SystemMessagePromptTemplate, -) -from langchain.schema import HumanMessage, SystemMessage - -chat = ChatOpenAI( - openai_api_base="http://0.0.0.0:4000", - model = "gpt-3.5-turbo", - temperature=0.1, - extra_body={ - "metadata": { - "generation_name": "ishaan-generation-langchain-client", - "generation_id": "langchain-client-gen-id22", - "trace_id": "langchain-client-trace-id22", - "trace_user_id": "langchain-client-user-id2" - } - } -) - -messages = [ - SystemMessage( - content="You are a helpful assistant that im using to make a test request to." - ), - HumanMessage( - content="test from litellm. tell me why it's amazing in 1 sentence" - ), -] -response = chat(messages) - -print(response) -``` - - - - - -### Team based Logging to Langfuse - -**Example:** - -This config would send langfuse logs to 2 different langfuse projects, based on the team id - -```yaml -litellm_settings: - default_team_settings: - - team_id: my-secret-project - success_callback: ["langfuse"] - langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_1 # Project 1 - langfuse_secret: os.environ/LANGFUSE_PRIVATE_KEY_1 # Project 1 - - team_id: ishaans-secret-project - success_callback: ["langfuse"] - langfuse_public_key: os.environ/LANGFUSE_PUB_KEY_2 # Project 2 - langfuse_secret: os.environ/LANGFUSE_SECRET_2 # Project 2 -``` - -Now, when you [generate keys](./virtual_keys.md) for this team-id - -```bash -curl -X POST 'http://0.0.0.0:4000/key/generate' \ --H 'Authorization: Bearer sk-1234' \ --H 'Content-Type: application/json' \ --d '{"team_id": "ishaans-secret-project"}' -``` - -All requests made with these keys will log data to their team-specific logging. - -### Redacting Messages, Response Content from Langfuse Logging - -Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to langfuse, but request metadata will still be logged. - -```yaml -model_list: - - model_name: gpt-3.5-turbo - litellm_params: - model: gpt-3.5-turbo -litellm_settings: - success_callback: ["langfuse"] - turn_off_message_logging: True -``` - - - ## Logging Proxy Cost + Usage - OpenMeter Bill customers according to their LLM API usage with [OpenMeter](../observability/openmeter.md)