From e394ed1e5b251e6013fb930acb5bdc7437c506e3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 4 Oct 2024 16:58:04 +0530 Subject: [PATCH] (fixes) docs + qa - gcs key based logging (#6061) * fixes for required values for gcs bucket * docs gcs bucket logging --- docs/my-website/docs/proxy/team_logging.md | 88 ++++++++++++++++------ litellm/integrations/gcs_bucket.py | 30 ++++++-- litellm/integrations/gcs_bucket_base.py | 10 +-- 3 files changed, 91 insertions(+), 37 deletions(-) diff --git a/docs/my-website/docs/proxy/team_logging.md b/docs/my-website/docs/proxy/team_logging.md index ed5d28af467..d96efaa8b24 100644 --- a/docs/my-website/docs/proxy/team_logging.md +++ b/docs/my-website/docs/proxy/team_logging.md @@ -86,12 +86,14 @@ curl -X POST 'http:/localhost:4000/team/dbe2f686-a686-4896-864a-4c3924458709/cal | Field | Supported Values | Notes | |-------|------------------|-------| -| `callback_name` | `"langfuse"` | Currently only supports "langfuse" | +| `callback_name` | `"langfuse"`, `"gcs_bucket"`| Currently only supports `"langfuse"`, `"gcs_bucket"` | | `callback_type` | `"success"`, `"failure"`, `"success_and_failure"` | | | `callback_vars` | | dict of callback settings | -|     `langfuse_public_key` | string | Required | -|     `langfuse_secret_key` | string | Required | -|     `langfuse_host` | string | Optional (defaults to https://cloud.langfuse.com) | +|     `langfuse_public_key` | string | Required for Langfuse | +|     `langfuse_secret_key` | string | Required for Langfuse | +|     `langfuse_host` | string | Optional for Langfuse (defaults to https://cloud.langfuse.com) | +|     `gcs_bucket_name` | string | Required for GCS Bucket. Name of your GCS bucket | +|     `gcs_path_service_account` | string | Required for GCS Bucket. Path to your service account json | #### 2. Create key for team @@ -229,24 +231,68 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ -```bash -curl -X POST 'http://0.0.0.0:4000/key/generate' \ --H 'Authorization: Bearer sk-1234' \ --H 'Content-Type: application/json' \ --d '{ - "metadata": { - "logging": [{ - "callback_name": "gcs_bucket", # "otel", "gcs_bucket" - "callback_type": "success", # "success", "failure", "success_and_failure" - "callback_vars": { - "gcs_bucket_name": "my-gcs-bucket", - "gcs_path_service_account": "os.environ/GCS_SERVICE_ACCOUNT" - } - }] - } -}' +1. Set gcs_bucket logging on your config.yaml -``` + By default, this will use + - `GCS_PATH_SERVICE_ACCOUNT` env variable to get the service account json path + - `GCS_BUCKET_NAME` env variable to get the bucket name to send logs to + + ```yaml + model_list: + - model_name: fake-openai-endpoint + litellm_params: + model: openai/fake + api_key: fake-key + api_base: https://exampleopenaiendpoint-production.up.railway.app/ + + litellm_settings: + callbacks: ["gcs_bucket"] + ``` + +2. Create Virtual Key to log to a specific GCS Bucket + + Set `GCS_SERVICE_ACCOUNT` in your environment to the path of the service account json + ```bash + export GCS_SERVICE_ACCOUNT=/path/to/service-account.json # GCS_SERVICE_ACCOUNT=/Users/ishaanjaffer/Downloads/adroit-crow-413218-a956eef1a2a8.json + ``` + + ```bash + curl -X POST 'http://0.0.0.0:4000/key/generate' \ + -H 'Authorization: Bearer sk-1234' \ + -H 'Content-Type: application/json' \ + -d '{ + "metadata": { + "logging": [{ + "callback_name": "gcs_bucket", # "otel", "gcs_bucket" + "callback_type": "success", # "success", "failure", "success_and_failure" + "callback_vars": { + "gcs_bucket_name": "my-gcs-bucket", # Name of your GCS Bucket to log to + "gcs_path_service_account": "os.environ/GCS_SERVICE_ACCOUNT" # environ variable for this service account + } + }] + } + }' + + ``` + +4. Test it - `/chat/completions` request + + Use the virtual key from step 3 to make a `/chat/completions` request + + You should see your logs on GCS Bucket on a successful request + + ```shell + curl -i http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-Fxq5XSyWKeXDKfPdqXZhPg" \ + -d '{ + "model": "fake-openai-endpoint", + "messages": [ + {"role": "user", "content": "Hello, Claude"} + ], + "user": "hello", + }' + ``` diff --git a/litellm/integrations/gcs_bucket.py b/litellm/integrations/gcs_bucket.py index 4d82bd56b20..2c9da558416 100644 --- a/litellm/integrations/gcs_bucket.py +++ b/litellm/integrations/gcs_bucket.py @@ -46,12 +46,6 @@ class GCSBucketLogger(GCSBucketBase): f"GCS Bucket logging is a premium feature. Please upgrade to use it. {CommonProxyErrors.not_premium_user.value}" ) - if self.path_service_account_json is None: - raise ValueError( - "GCS_PATH_SERVICE_ACCOUNT is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_PATH_SERVICE_ACCOUNT' in the environment." - ) - pass - #### ASYNC #### async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): from litellm.proxy.proxy_server import premium_user @@ -178,26 +172,46 @@ class GCSBucketLogger(GCSBucketBase): kwargs.get("standard_callback_dynamic_params", None) ) + bucket_name: str + path_service_account: str if standard_callback_dynamic_params is not None: verbose_logger.debug("Using dynamic GCS logging") verbose_logger.debug( "standard_callback_dynamic_params: %s", standard_callback_dynamic_params ) - bucket_name: str = ( + _bucket_name: Optional[str] = ( standard_callback_dynamic_params.get("gcs_bucket_name", None) or self.BUCKET_NAME ) - path_service_account: str = ( + _path_service_account: Optional[str] = ( standard_callback_dynamic_params.get("gcs_path_service_account", None) or self.path_service_account_json ) + if _bucket_name is None: + raise ValueError( + "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." + ) + if _path_service_account is None: + raise ValueError( + "GCS_PATH_SERVICE_ACCOUNT is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_PATH_SERVICE_ACCOUNT' in the environment." + ) + bucket_name = _bucket_name + path_service_account = _path_service_account vertex_instance = await self.get_or_create_vertex_instance( credentials=path_service_account ) else: # If no dynamic parameters, use the default instance + if self.BUCKET_NAME is None: + raise ValueError( + "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." + ) + if self.path_service_account_json is None: + raise ValueError( + "GCS_PATH_SERVICE_ACCOUNT is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_PATH_SERVICE_ACCOUNT' in the environment." + ) bucket_name = self.BUCKET_NAME path_service_account = self.path_service_account_json vertex_instance = await self.get_or_create_vertex_instance( diff --git a/litellm/integrations/gcs_bucket_base.py b/litellm/integrations/gcs_bucket_base.py index 1d1672c4dc7..b9e0930be7d 100644 --- a/litellm/integrations/gcs_bucket_base.py +++ b/litellm/integrations/gcs_bucket_base.py @@ -31,14 +31,8 @@ class GCSBucketBase(CustomLogger): ) _path_service_account = os.getenv("GCS_PATH_SERVICE_ACCOUNT") _bucket_name = bucket_name or os.getenv("GCS_BUCKET_NAME") - if _path_service_account is None: - raise ValueError("GCS_PATH_SERVICE_ACCOUNT environment variable is not set") - if _bucket_name is None: - raise ValueError( - "GCS_BUCKET_NAME is not set in the environment, but GCS Bucket is being used as a logging callback. Please set 'GCS_BUCKET_NAME' in the environment." - ) - self.path_service_account_json: str = _path_service_account - self.BUCKET_NAME: str = _bucket_name + self.path_service_account_json: Optional[str] = _path_service_account + self.BUCKET_NAME: Optional[str] = _bucket_name async def construct_request_headers( self,