diff --git a/cookbook/logging_observability/LiteLLM_Arize.ipynb b/cookbook/logging_observability/LiteLLM_Arize.ipynb index 82dfc1ceff1..72a082f874d 100644 --- a/cookbook/logging_observability/LiteLLM_Arize.ipynb +++ b/cookbook/logging_observability/LiteLLM_Arize.ipynb @@ -110,11 +110,6 @@ "os.environ['OPENAI_API_KEY']= getpass(\"Enter your OpenAI API key: \")" ] }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, { "cell_type": "markdown", "metadata": {}, diff --git a/docs/my-website/docs/observability/arize_integration.md b/docs/my-website/docs/observability/arize_integration.md index 62cf8404c0e..1cd36a1111f 100644 --- a/docs/my-website/docs/observability/arize_integration.md +++ b/docs/my-website/docs/observability/arize_integration.md @@ -11,12 +11,12 @@ https://github.com/BerriAI/litellm ::: -## Pre-Requisites + +## Pre-Requisites Make an account on [Arize AI](https://app.arize.com/auth/login) ## Quick Start - Use just 2 lines of code, to instantly log your responses **across all providers** with arize You can also use the instrumentor option instead of the callback, which you can find [here](https://docs.arize.com/arize/llm-tracing/tracing-integrations-auto/litellm). @@ -24,7 +24,6 @@ You can also use the instrumentor option instead of the callback, which you can ```python litellm.callbacks = ["arize"] ``` - ```python import litellm import os @@ -37,7 +36,7 @@ os.environ['OPENAI_API_KEY']="" # set arize as a callback, litellm will send the data to arize litellm.callbacks = ["arize"] - + # openai call response = litellm.completion( model="gpt-3.5-turbo", @@ -49,6 +48,7 @@ response = litellm.completion( ### Using with LiteLLM Proxy + ```yaml model_list: - model_name: gpt-4 @@ -61,10 +61,10 @@ litellm_settings: callbacks: ["arize"] environment_variables: - ARIZE_SPACE_KEY: "d0*****" - ARIZE_API_KEY: "141a****" - ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint - ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT + ARIZE_SPACE_KEY: "d0*****" + ARIZE_API_KEY: "141a****" + ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint + ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT ``` ## Support & Talk to Founders diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index 6f2f7250b4c..e13a4036344 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -17,6 +17,8 @@ Log Proxy input, output, and exceptions using: - DynamoDB - etc. + + ## Getting the LiteLLM Call ID LiteLLM generates a unique `call_id` for each request. This `call_id` can be @@ -50,29 +52,31 @@ A number of these headers could be useful for troubleshooting, but the `x-litellm-call-id` is the one that is most useful for tracking a request across components in your system, including in logging tools. + ## Logging Features ### Conditional Logging by Virtual Keys, Teams Use this to: - 1. Conditionally enable logging for some virtual keys/teams 2. Set different logging providers for different virtual keys/teams [👉 **Get Started** - Team/Key Based Logging](team_logging) -### Redacting UserAPIKeyInfo -Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. +### Redacting UserAPIKeyInfo + +Redact information about the user api key (hashed token, user_id, team id, etc.), from logs. Currently supported for Langfuse, OpenTelemetry, Logfire, ArizeAI logging. ```yaml -litellm_settings: +litellm_settings: callbacks: ["langfuse"] redact_user_api_key_info: true ``` + ### Redact Messages, Response Content Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to your logging provider, but request metadata - e.g. spend, will still be tracked. @@ -82,7 +86,6 @@ Set `litellm.turn_off_message_logging=True` This will prevent the messages and r **1. Setup config.yaml ** - ```yaml model_list: - model_name: gpt-3.5-turbo @@ -94,7 +97,6 @@ litellm_settings: ``` **2. Send request** - ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ @@ -109,12 +111,14 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` + + :::info -Dynamic request message redaction is in BETA. +Dynamic request message redaction is in BETA. ::: @@ -166,11 +170,13 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ + ### Disable Message Redaction If you have `litellm.turn_on_message_logging` turned on, you can override it for specific requests by setting a request header `LiteLLM-Disable-Message-Redaction: true`. + ```shell curl --location 'http://0.0.0.0:4000/chat/completions' \ --header 'Content-Type: application/json' \ @@ -186,6 +192,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` + ### Turn off all tracking/logging For some use cases, you may want to turn off all tracking/logging. You can do this by passing `no-log=True` in the request body. @@ -198,7 +205,6 @@ Disable this by setting `global_disable_no_log_param:true` in your config.yaml f litellm_settings: global_disable_no_log_param: True ``` - ::: @@ -256,12 +262,13 @@ print(response) -**Expected Console Log** +**Expected Console Log** ``` LiteLLM.Info: "no-log request, skipping logging" ``` + ## What gets logged? Found under `kwargs["standard_logging_object"]`. This is a standard payload, logged for every response. @@ -424,8 +431,10 @@ print(response) Set `tags` as part of your request body + + ```python @@ -453,7 +462,6 @@ response = client.chat.completions.create( print(response) ``` - @@ -478,7 +486,6 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } }' ``` - @@ -521,26 +528,28 @@ print(response) + + ### LiteLLM Tags - `cache_hit`, `cache_key` Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields -| LiteLLM specific field | Description | Example Value | -| ------------------------- | --------------------------------------------------------------------------------------- | --------------------------------------- | -| `cache_hit` | Indicates whether a cache hit occured (True) or not (False) | `true`, `false` | -| `cache_key` | The Cache key used for this request | `d2b758c****` | -| `proxy_base_url` | The base URL for the proxy server, the value of env var `PROXY_BASE_URL` on your server | `https://proxy.example.com` | -| `user_api_key_alias` | An alias for the LiteLLM Virtual Key. | `prod-app1` | -| `user_api_key_user_id` | The unique ID associated with a user's API key. | `user_123`, `user_456` | -| `user_api_key_user_email` | The email associated with a user's API key. | `user@example.com`, `admin@example.com` | -| `user_api_key_team_alias` | An alias for a team associated with an API key. | `team_alpha`, `dev_team` | +| LiteLLM specific field | Description | Example Value | +|------------------------|-------------------------------------------------------|------------------------------------------------| +| `cache_hit` | Indicates whether a cache hit occured (True) or not (False) | `true`, `false` | +| `cache_key` | The Cache key used for this request | `d2b758c****`| +| `proxy_base_url` | The base URL for the proxy server, the value of env var `PROXY_BASE_URL` on your server | `https://proxy.example.com`| +| `user_api_key_alias` | An alias for the LiteLLM Virtual Key.| `prod-app1` | +| `user_api_key_user_id` | The unique ID associated with a user's API key. | `user_123`, `user_456` | +| `user_api_key_user_email` | The email associated with a user's API key. | `user@example.com`, `admin@example.com` | +| `user_api_key_team_alias` | An alias for a team associated with an API key. | `team_alpha`, `dev_team` | + **Usage** Specify `langfuse_default_tags` to control what litellm fields get logged on Langfuse -Example config.yaml - +Example config.yaml ```yaml model_list: - model_name: gpt-4 @@ -553,23 +562,12 @@ litellm_settings: success_callback: ["langfuse"] # 👇 Key Change - langfuse_default_tags: - [ - "cache_hit", - "cache_key", - "proxy_base_url", - "user_api_key_alias", - "user_api_key_user_id", - "user_api_key_user_email", - "user_api_key_team_alias", - "semantic-similarity", - "proxy_base_url", - ] + langfuse_default_tags: ["cache_hit", "cache_key", "proxy_base_url", "user_api_key_alias", "user_api_key_user_id", "user_api_key_user_email", "user_api_key_team_alias", "semantic-similarity", "proxy_base_url"] ``` ### View POST sent from LiteLLM to provider -Use this when you want to view the RAW curl request sent from LiteLLM to the LLM API +Use this when you want to view the RAW curl request sent from LiteLLM to the LLM API @@ -672,7 +670,7 @@ You will see `raw_request` in your Langfuse Metadata. This is the RAW CURL comma ## OpenTelemetry -:::info +:::info [Optional] Customize OTEL Service Name and OTEL TRACER NAME by setting the following variables in your environment @@ -732,30 +730,30 @@ This is the Span from OTEL Logging ```json { - "name": "litellm-acompletion", - "context": { - "trace_id": "0x8d354e2346060032703637a0843b20a3", - "span_id": "0xd8d3476a2eb12724", - "trace_state": "[]" - }, - "kind": "SpanKind.INTERNAL", - "parent_id": null, - "start_time": "2024-06-04T19:46:56.415888Z", - "end_time": "2024-06-04T19:46:56.790278Z", - "status": { - "status_code": "OK" - }, - "attributes": { - "model": "llama3-8b-8192" - }, - "events": [], - "links": [], - "resource": { - "attributes": { - "service.name": "litellm" + "name": "litellm-acompletion", + "context": { + "trace_id": "0x8d354e2346060032703637a0843b20a3", + "span_id": "0xd8d3476a2eb12724", + "trace_state": "[]" }, - "schema_url": "" - } + "kind": "SpanKind.INTERNAL", + "parent_id": null, + "start_time": "2024-06-04T19:46:56.415888Z", + "end_time": "2024-06-04T19:46:56.790278Z", + "status": { + "status_code": "OK" + }, + "attributes": { + "model": "llama3-8b-8192" + }, + "events": [], + "links": [], + "resource": { + "attributes": { + "service.name": "litellm" + }, + "schema_url": "" + } } ``` @@ -967,7 +965,6 @@ callback_settings: ``` ### Traceparent Header - ##### Context propagation across Services `Traceparent HTTP Header` ❓ Use this when you want to **pass information about the incoming request in a distributed tracing system** @@ -1045,23 +1042,25 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? ::: -| Property | Details | -| ---------------------------- | -------------------------------------------------------------- | -| Description | Log LLM Input/Output to cloud storage buckets | -| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) | + +| Property | Details | +|----------|---------| +| Description | Log LLM Input/Output to cloud storage buckets | +| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) | | Google Docs on Cloud Storage | [Google Cloud Storage](https://cloud.google.com/storage?hl=en) | + + #### Usage 1. Add `gcs_bucket` to LiteLLM Config.yaml - ```yaml model_list: - - litellm_params: - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - api_key: my-fake-key - model: openai/my-fake-model - model_name: fake-openai-endpoint +- litellm_params: + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint litellm_settings: callbacks: ["gcs_bucket"] # 👈 KEY CHANGE # 👈 KEY CHANGE @@ -1080,7 +1079,7 @@ GCS_PATH_SERVICE_ACCOUNT="/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956e litellm --config /path/to/config.yaml ``` -4. Test it! +4. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -1097,6 +1096,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` + #### Expected Logs on GCS Buckets @@ -1105,6 +1105,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ [**The standard logging object is logged on GCS Bucket**](../proxy/logging_spec) + #### Getting `service_account.json` from Google Cloud Console 1. Go to [Google Cloud Console](https://console.cloud.google.com/) @@ -1114,6 +1115,8 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ 5. Click on 'Keys' -> Add Key -> Create New Key -> JSON 6. Save the JSON file and add the path to `GCS_PATH_SERVICE_ACCOUNT` + + ## Google Cloud Storage - PubSub Topic Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.google.com/pubsub/docs/reference/rest) @@ -1124,25 +1127,26 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog ::: -| Property | Details | -| ----------- | ------------------------------------------------------------------ | + +| Property | Details | +|----------|---------| | Description | Log LiteLLM `SpendLogs Table` to Google Cloud Storage PubSub Topic | When to use `gcs_pubsub`? - If your LiteLLM Database has crossed 1M+ spend logs and you want to send `SpendLogs` to a PubSub Topic that can be consumed by GCS BigQuery + #### Usage 1. Add `gcs_pubsub` to LiteLLM Config.yaml - ```yaml model_list: - - litellm_params: - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - api_key: my-fake-key - model: openai/my-fake-model - model_name: fake-openai-endpoint +- litellm_params: + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint litellm_settings: callbacks: ["gcs_pubsub"] # 👈 KEY CHANGE # 👈 KEY CHANGE @@ -1161,7 +1165,7 @@ GCS_PUBSUB_PROJECT_ID="reliableKeys" litellm --config /path/to/config.yaml ``` -4. Test it! +4. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -1178,11 +1182,13 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` + + ## s3 Buckets -We will use the `--config` to set +We will use the `--config` to set -- `litellm.success_callback = ["s3"]` +- `litellm.success_callback = ["s3"]` This will log all successfull LLM calls to s3 Bucket @@ -1270,15 +1276,17 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur ::: -| Property | Details | -| ------------------------------- | --------------------------------------------------------------------------------------------------------------- | -| Description | Log LLM Input/Output to Azure Blob Storag (Bucket) | + +| Property | Details | +|----------|---------| +| Description | Log LLM Input/Output to Azure Blob Storag (Bucket) | | Azure Docs on Data Lake Storage | [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction) | + + #### Usage 1. Add `azure_storage` to LiteLLM Config.yaml - ```yaml model_list: - model_name: fake-openai-endpoint @@ -1314,7 +1322,7 @@ AZURE_STORAGE_CLIENT_SECRET="uMS8Qxxxxxxxxxx" # The Application Client Secret to litellm --config /path/to/config.yaml ``` -4. Test it! +4. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -1331,6 +1339,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ' ``` + #### Expected Logs on Azure Data Lake Storage @@ -1339,10 +1348,11 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ [**The standard logging object is logged on Azure Data Lake Storage**](../proxy/logging_spec) + + ## DataDog LiteLLM Supports logging to the following Datdog Integrations: - - `datadog` [Datadog Logs](https://docs.datadoghq.com/logs/) - `datadog_llm_observability` [Datadog LLM Observability](https://www.datadoghq.com/product/llm-observability/) - `ddtrace-run` [Datadog Tracing](#datadog-tracing) @@ -1437,29 +1447,26 @@ docker run \ LiteLLM supports customizing the following Datadog environment variables -| Environment Variable | Description | Default Value | Required | -| -------------------- | ------------------------------------------------------------- | ---------------- | -------- | -| `DD_API_KEY` | Your Datadog API key for authentication | None | ✅ Yes | -| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") | None | ✅ Yes | -| `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | -| `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | -| `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | -| `DD_VERSION` | Version tag for your logs | "unknown" | ❌ No | -| `HOSTNAME` | Hostname tag for your logs | "" | ❌ No | -| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | +| Environment Variable | Description | Default Value | Required | +|---------------------|-------------|---------------|----------| +| `DD_API_KEY` | Your Datadog API key for authentication | None | ✅ Yes | +| `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") | None | ✅ Yes | +| `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | +| `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | +| `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | +| `DD_VERSION` | Version tag for your logs | "unknown" | ❌ No | +| `HOSTNAME` | Hostname tag for your logs | "" | ❌ No | +| `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | + ## Lunary - -#### Step1: Install dependencies and set your environment variables - +#### Step1: Install dependencies and set your environment variables Install the dependencies - ```shell pip install litellm lunary ``` -Get you Lunary public key from from https://app.lunary.ai/settings - +Get you Lunary public key from from https://app.lunary.ai/settings ```shell export LUNARY_PUBLIC_KEY="" ``` @@ -1477,7 +1484,6 @@ litellm_settings: ``` #### Step 3: Start the LiteLLM proxy - ```shell litellm --config config.yaml ``` @@ -1504,8 +1510,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ ## MLflow -#### Step1: Install dependencies +#### Step1: Install dependencies Install the dependencies. ```shell @@ -1525,7 +1531,6 @@ litellm_settings: ``` #### Step 3: Start the LiteLLM proxy - ```shell litellm --config config.yaml ``` @@ -1554,6 +1559,8 @@ Run the following command to start MLflow UI and review recorded traces. mlflow ui ``` + + ## Custom Callback Class [Async] Use this when you want to run custom callbacks in `python` @@ -1564,7 +1571,7 @@ We use `litellm.integrations.custom_logger` for this, **more details about litel Define your custom callback class in a python file. -Here's an example custom logger for tracking `key, user, model, prompt, response, tokens, cost`. We create a file called `custom_callbacks.py` and initialize `proxy_handler_instance` +Here's an example custom logger for tracking `key, user, model, prompt, response, tokens, cost`. We create a file called `custom_callbacks.py` and initialize `proxy_handler_instance` ```python from litellm.integrations.custom_logger import CustomLogger @@ -1573,16 +1580,16 @@ import litellm # This file includes the custom callbacks for LiteLLM Proxy # Once defined, these can be passed in proxy_config.yaml class MyCustomHandler(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): + def log_pre_api_call(self, model, messages, kwargs): print(f"Pre-API Call") - - def log_post_api_call(self, kwargs, response_obj, start_time, end_time): + + def log_post_api_call(self, kwargs, response_obj, start_time, end_time): print(f"Post-API Call") - - def log_success_event(self, kwargs, response_obj, start_time, end_time): + + def log_success_event(self, kwargs, response_obj, start_time, end_time): print("On Success") - def log_failure_event(self, kwargs, response_obj, start_time, end_time): + def log_failure_event(self, kwargs, response_obj, start_time, end_time): print(f"On Failure") async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): @@ -1600,7 +1607,7 @@ class MyCustomHandler(CustomLogger): # Calculate cost using litellm.completion_cost() cost = litellm.completion_cost(completion_response=response_obj) response = response_obj - # tokens used in response + # tokens used in response usage = response_obj["usage"] print( @@ -1616,7 +1623,7 @@ class MyCustomHandler(CustomLogger): ) return - async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): try: print(f"On Async Failure !") print("\nkwargs", kwargs) @@ -1636,7 +1643,7 @@ class MyCustomHandler(CustomLogger): # Calculate cost using litellm.completion_cost() cost = litellm.completion_cost(completion_response=response_obj) print("now checking response obj") - + print( f""" Model: {model}, @@ -1660,7 +1667,7 @@ proxy_handler_instance = MyCustomHandler() #### Step 2 - Pass your custom callback class in `config.yaml` -We pass the custom callback class defined in **Step1** to the config.yaml. +We pass the custom callback class defined in **Step1** to the config.yaml. Set `callbacks` to `python_filename.logger_instance_name` In the config below, we pass @@ -1678,6 +1685,7 @@ model_list: litellm_settings: callbacks: custom_callbacks.proxy_handler_instance # sets litellm.callbacks = [proxy_handler_instance] + ``` #### Step 3 - Start proxy + test request @@ -1758,7 +1766,7 @@ class MyCustomHandler(CustomLogger): } ``` -#### Logging `model_info` set in config.yaml +#### Logging `model_info` set in config.yaml Here is how to log the `model_info` set in your proxy `config.yaml`. Information on setting `model_info` on [config.yaml](https://docs.litellm.ai/docs/proxy/configs) @@ -1775,7 +1783,7 @@ class MyCustomHandler(CustomLogger): **Expected Output** ```json -{ "mode": "embedding", "input_cost_per_token": 0.002 } +{'mode': 'embedding', 'input_cost_per_token': 0.002} ``` ##### Logging responses from proxy @@ -1855,7 +1863,7 @@ Use this if you: #### Step 1. Create your generic logging API endpoint -Set up a generic API endpoint that can receive data in JSON format. The data will be included within a "data" field. +Set up a generic API endpoint that can receive data in JSON format. The data will be included within a "data" field. Your server should support the following Request format: @@ -1938,7 +1946,7 @@ litellm_settings: success_callback: ["generic"] ``` -Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API +Start the LiteLLM Proxy and make a test request to verify the logs reached your callback API ## Langsmith @@ -1958,13 +1966,14 @@ environment_variables: LANGSMITH_BASE_URL: "https://api.smith.langchain.com" # (Optional - only needed if you have a custom Langsmith instance) ``` + 2. Start Proxy ``` litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -1980,10 +1989,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } ' ``` - Expect to see your log on Langfuse + ## Arize AI 1. Set `success_callback: ["arize"]` on litellm config.yaml @@ -2000,10 +2009,10 @@ litellm_settings: callbacks: ["arize"] environment_variables: - ARIZE_SPACE_KEY: "d0*****" - ARIZE_API_KEY: "141a****" - ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint - ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT + ARIZE_SPACE_KEY: "d0*****" + ARIZE_API_KEY: "141a****" + ARIZE_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize GRPC api endpoint + ARIZE_HTTP_ENDPOINT: "https://otlp.arize.com/v1" # OPTIONAL - your custom arize HTTP api endpoint. Set either this or ARIZE_ENDPOINT ``` 2. Start Proxy @@ -2012,7 +2021,7 @@ environment_variables: litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -2028,10 +2037,10 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ } ' ``` - Expect to see your log on Langfuse + ## Langtrace 1. Set `success_callback: ["langtrace"]` on litellm config.yaml @@ -2048,7 +2057,7 @@ litellm_settings: callbacks: ["langtrace"] environment_variables: - LANGTRACE_API_KEY: "141a****" + LANGTRACE_API_KEY: "141a****" ``` 2. Start Proxy @@ -2057,7 +2066,7 @@ environment_variables: litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -2095,17 +2104,17 @@ export GALILEO_USERNAME="" export GALILEO_PASSWORD="" ``` -#### Quick Start +#### Quick Start 1. Add to Config.yaml ```yaml model_list: - - litellm_params: - api_base: https://exampleopenaiendpoint-production.up.railway.app/ - api_key: my-fake-key - model: openai/my-fake-model - model_name: fake-openai-endpoint +- litellm_params: + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint litellm_settings: success_callback: ["galileo"] # 👈 KEY CHANGE @@ -2117,7 +2126,7 @@ litellm_settings: litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -2148,17 +2157,17 @@ export OPENMETER_API_ENDPOINT="" # defaults to https://openmeter.cloud export OPENMETER_API_KEY="" ``` -##### Quick Start +##### Quick Start 1. Add to Config.yaml ```yaml model_list: - - litellm_params: - api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ - api_key: my-fake-key - model: openai/my-fake-model - model_name: fake-openai-endpoint +- litellm_params: + api_base: https://openai-function-calling-workers.tasslexyz.workers.dev/ + api_key: my-fake-key + model: openai/my-fake-model + model_name: fake-openai-endpoint litellm_settings: success_callback: ["openmeter"] # 👈 KEY CHANGE @@ -2170,7 +2179,7 @@ litellm_settings: litellm --config /path/to/config.yaml ``` -3. Test it! +3. Test it! ```bash curl --location 'http://0.0.0.0:4000/chat/completions' \ @@ -2191,9 +2200,9 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ## DynamoDB -We will use the `--config` to set +We will use the `--config` to set -- `litellm.success_callback = ["dynamodb"]` +- `litellm.success_callback = ["dynamodb"]` - `litellm.dynamodb_table_name = "your-table-name"` This will log all successfull LLM calls to DynamoDB @@ -2317,7 +2326,7 @@ Your logs should be available on DynamoDB ## Sentry -If api calls fail (llm/database) you can log those to Sentry: +If api calls fail (llm/database) you can log those to Sentry: **Step 1** Install Sentry @@ -2331,7 +2340,7 @@ pip install --upgrade sentry-sdk export SENTRY_DSN="your-sentry-dsn" ``` -```yaml +```yaml model_list: - model_name: gpt-3.5-turbo litellm_params: @@ -2339,7 +2348,7 @@ model_list: litellm_settings: # other settings failure_callback: ["sentry"] -general_settings: +general_settings: database_url: "my-bad-url" # set a fake url to trigger a sentry exception ``` @@ -2404,6 +2413,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ }' ``` + +::: --> \ No newline at end of file diff --git a/litellm/types/integrations/arize.py b/litellm/types/integrations/arize.py index b1559aafa8e..24298fc3636 100644 --- a/litellm/types/integrations/arize.py +++ b/litellm/types/integrations/arize.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Literal, Any, Optional +from typing import TYPE_CHECKING, Literal, Any from pydantic import BaseModel @@ -9,6 +9,6 @@ else: class ArizeConfig(BaseModel): space_key: str - api_key: str + api_key: str protocol: Protocol endpoint: str diff --git a/tests/local_testing/test_arize_ai.py b/tests/local_testing/test_arize_ai.py index 309cc689ea8..cdc59115081 100644 --- a/tests/local_testing/test_arize_ai.py +++ b/tests/local_testing/test_arize_ai.py @@ -97,4 +97,5 @@ def test_arize_set_attributes(): span.set_attribute.assert_any_call("llm.output_messages.0.message.content", "response content") span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_TOTAL, 100) span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_COMPLETION, 60) - span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_PROMPT, 40) \ No newline at end of file + span.set_attribute.assert_any_call(SpanAttributes.LLM_TOKEN_COUNT_PROMPT, 40) + \ No newline at end of file