diff --git a/docs/my-website/docs/observability/braintrust.md b/docs/my-website/docs/observability/braintrust.md index 5a88964069d..79f3cf13be2 100644 --- a/docs/my-website/docs/observability/braintrust.md +++ b/docs/my-website/docs/observability/braintrust.md @@ -2,25 +2,24 @@ import Image from '@theme/IdealImage'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Braintrust - Evals + Logging +# Braintrust - Evals + Logging [Braintrust](https://www.braintrust.dev/) manages evaluations, logging, prompt playground, to data management for AI products. - ## Quick Start ```python -# pip install langfuse +# pip install braintrust import litellm import os -# set env -os.environ["BRAINTRUST_API_KEY"] = "" +# set env +os.environ["BRAINTRUST_API_KEY"] = "" os.environ['OPENAI_API_KEY']="" # set braintrust as a callback, litellm will send the data to braintrust -litellm.callbacks = ["braintrust"] - +litellm.callbacks = ["braintrust"] + # openai call response = litellm.completion( model="gpt-3.5-turbo", @@ -30,16 +29,16 @@ response = litellm.completion( ) ``` - - ## OpenAI Proxy Usage -1. Add keys to env +1. Add keys to env + ```env -BRAINTRUST_API_KEY="" +BRAINTRUST_API_KEY="" ``` -2. Add braintrust to callbacks +2. Add braintrust to callbacks + ```yaml model_list: - model_name: gpt-3.5-turbo @@ -47,12 +46,11 @@ model_list: model: gpt-3.5-turbo api_key: os.environ/OPENAI_API_KEY - litellm_settings: callbacks: ["braintrust"] ``` -3. Test it! +3. Test it! ```bash curl -X POST 'http://0.0.0.0:4000/chat/completions' \ @@ -69,6 +67,8 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ ## Advanced - pass Project ID or name +It is recommended that you include the `project_id` or `project_name` to ensure your traces are being written out to the correct Braintrust project. + @@ -77,12 +77,28 @@ response = litellm.completion( model="gpt-3.5-turbo", messages=[ {"role": "user", "content": "Hi 👋 - i'm openai"} - ], + ], metadata={ "project_id": "1234", # passing project_name will try to find a project with that name, or create one if it doesn't exist # if both project_id and project_name are passed, project_id will be used - # "project_name": "my-special-project" + # "project_name": "my-special-project" + } +) +``` + +Note: Other `metadata` can be included here as well when using the SDK. + +```python +response = litellm.completion( + model="gpt-3.5-turbo", + messages=[ + {"role": "user", "content": "Hi 👋 - i'm openai"} + ], + metadata={ + "project_id": "1234", + "item1": "an item", + "item2": "another item" } ) ``` @@ -127,7 +143,7 @@ response = client.chat.completions.create( } ], extra_body={ # pass in any provider-specific param, if not supported by openai, https://docs.litellm.ai/docs/completion/input#provider-specific-params - "metadata": { # 👈 use for logging additional params (e.g. to langfuse) + "metadata": { # 👈 use for logging additional params (e.g. to braintrust) "project_id": "my-special-project" } } @@ -141,10 +157,10 @@ For more examples, [**Click Here**](../proxy/user_keys.md#chatcompletions) -## Full API Spec +## Full API Spec -Here's everything you can pass in metadata for a braintrust request +Here's everything you can pass in metadata for a braintrust request -`braintrust_*` - any metadata field starting with `braintrust_` will be passed as metadata to the logging request +`braintrust_*` - If you are adding metadata from _proxy request headers_, any metadata field starting with `braintrust_` will be passed as metadata to the logging request. If you are using the SDK, just pass your metadata like normal (e.g., `metadata={"project_name": "my-test-project", "item1": "an item", "item2": "another item"}`) -`project_id` - set the project id for a braintrust call. Default is `litellm`. \ No newline at end of file +`project_id` - Set the project id for a braintrust call. Default is `litellm`. diff --git a/litellm/integrations/braintrust_logging.py b/litellm/integrations/braintrust_logging.py index 0961eab02b8..c68674f77ba 100644 --- a/litellm/integrations/braintrust_logging.py +++ b/litellm/integrations/braintrust_logging.py @@ -111,7 +111,7 @@ class BraintrustLogger(CustomLogger): @staticmethod def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: """ - Adds metadata from proxy request headers to Langfuse logging if keys start with "langfuse_" + Adds metadata from proxy request headers to Braintrust logging if keys start with "braintrust_" and overwrites litellm_params.metadata if already included. For example if you want to append your trace to an existing `trace_id` via header, send @@ -254,6 +254,11 @@ class BraintrustLogger(CustomLogger): if cost is not None: clean_metadata["litellm_response_cost"] = cost + # metadata.model is required for braintrust to calculate the "Estimated cost" metric + litellm_model = kwargs.get("model", None) + if litellm_model is not None: + clean_metadata["model"] = litellm_model + metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) if usage_obj and isinstance(usage_obj, litellm.Usage): @@ -391,6 +396,11 @@ class BraintrustLogger(CustomLogger): if cost is not None: clean_metadata["litellm_response_cost"] = cost + # metadata.model is required for braintrust to calculate the "Estimated cost" metric + litellm_model = kwargs.get("model", None) + if litellm_model is not None: + clean_metadata["model"] = litellm_model + metrics: Optional[dict] = None usage_obj = getattr(response_obj, "usage", None) if usage_obj and isinstance(usage_obj, litellm.Usage):