From 1cfc4624c3022652e542a09d6a3fa185d5561e36 Mon Sep 17 00:00:00 2001 From: Thomas Mildner <12685945+Thomas-Mildner@users.noreply.github.com> Date: Wed, 22 Oct 2025 01:40:55 +0200 Subject: [PATCH] [Feat] Add SENTRY_ENVIRONMENT configuration for Sentry integration (#15760) * [Feat] Add SENTRY_ENVIRONMENT configuration for Sentry integration and corresponding tests * [Refactor] Enhance test_sentry_environment by mocking sentry_sdk and improving environment handling * [Fix] Update default SENTRY_ENVIRONMENT to 'production' and enhance test for Sentry integration * [Fix] Update test_sentry_environment to verify correct handling of SENTRY_ENVIRONMENT values * [Fix] Update test_sentry_environment to assert correct handling of production environment --- docs/my-website/docs/observability/sentry.md | 6 ++ docs/my-website/docs/proxy/logging.md | 59 +++++++-------- litellm/litellm_core_utils/litellm_logging.py | 17 +++-- .../test_litellm_logging.py | 74 +++++++++++++++++++ 4 files changed, 119 insertions(+), 37 deletions(-) diff --git a/docs/my-website/docs/observability/sentry.md b/docs/my-website/docs/observability/sentry.md index b7992e35c54..46b19331b24 100644 --- a/docs/my-website/docs/observability/sentry.md +++ b/docs/my-website/docs/observability/sentry.md @@ -61,6 +61,12 @@ print(response) These options are useful for high-volume applications where sampling a subset of errors and transactions provides sufficient visibility while managing costs. +#### Sentry Environment +- **SENTRY_ENVIRONMENT**: Specifies the environment name for your Sentry events (e.g., "production", "staging", "development") + - Helps organize and filter errors by deployment environment in Sentry dashboard + - Example: `os.environ["SENTRY_ENVIRONMENT"] = "staging"` + - If not set, Sentry will use 'production' as the default environment + ## Redacting Messages, Response Content from Sentry Logging Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to sentry, but request metadata will still be logged. diff --git a/docs/my-website/docs/proxy/logging.md b/docs/my-website/docs/proxy/logging.md index ff2591daad2..497e6e95a52 100644 --- a/docs/my-website/docs/proxy/logging.md +++ b/docs/my-website/docs/proxy/logging.md @@ -602,15 +602,15 @@ print(response) Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields -| LiteLLM specific field | Description | Example Value | -|---------------------------|-----------------------------------------------------------------------------------------|------------------------------------------------| -| `cache_hit` | Indicates whether a cache hit occurred (True) or not (False) | `true`, `false` | -| `cache_key` | The Cache key used for this request | `d2b758c****` | -| `proxy_base_url` | The base URL for the proxy server, the value of env var `PROXY_BASE_URL` on your server | `https://proxy.example.com` | -| `user_api_key_alias` | An alias for the LiteLLM Virtual Key. | `prod-app1` | -| `user_api_key_user_id` | The unique ID associated with a user's API key. | `user_123`, `user_456` | -| `user_api_key_user_email` | The email associated with a user's API key. | `user@example.com`, `admin@example.com` | -| `user_api_key_team_alias` | An alias for a team associated with an API key. | `team_alpha`, `dev_team` | +| LiteLLM specific field | Description | Example Value | +| ------------------------- | --------------------------------------------------------------------------------------- | --------------------------------------- | +| `cache_hit` | Indicates whether a cache hit occurred (True) or not (False) | `true`, `false` | +| `cache_key` | The Cache key used for this request | `d2b758c****` | +| `proxy_base_url` | The base URL for the proxy server, the value of env var `PROXY_BASE_URL` on your server | `https://proxy.example.com` | +| `user_api_key_alias` | An alias for the LiteLLM Virtual Key. | `prod-app1` | +| `user_api_key_user_id` | The unique ID associated with a user's API key. | `user_123`, `user_456` | +| `user_api_key_user_email` | The email associated with a user's API key. | `user@example.com`, `admin@example.com` | +| `user_api_key_team_alias` | An alias for a team associated with an API key. | `team_alpha`, `dev_team` | **Usage** @@ -1111,10 +1111,10 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage? ::: -| Property | Details | -|----------|---------| -| Description | Log LLM Input/Output to cloud storage buckets | -| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) | +| Property | Details | +| ---------------------------- | -------------------------------------------------------------- | +| Description | Log LLM Input/Output to cloud storage buckets | +| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) | | Google Docs on Cloud Storage | [Google Cloud Storage](https://cloud.google.com/storage?hl=en) | @@ -1196,8 +1196,8 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog ::: -| Property | Details | -|----------|---------| +| Property | Details | +| ----------- | ------------------------------------------------------------------ | | Description | Log LiteLLM `SpendLogs Table` to Google Cloud Storage PubSub Topic | When to use `gcs_pubsub`? @@ -1388,10 +1388,10 @@ On s3 bucket, you will see the object key as `my-test-path/my-team-alias/...` ## AWS SQS -| Property | Details | -|----------|---------| -| Description | Log LLM Input/Output to AWS SQS Queue | -| AWS Docs on SQS | [AWS SQS](https://aws.amazon.com/sqs/) | +| Property | Details | +| -------------------- | ------------------------------------------------------------------------------------- | +| Description | Log LLM Input/Output to AWS SQS Queue | +| AWS Docs on SQS | [AWS SQS](https://aws.amazon.com/sqs/) | | Fields Logged to SQS | LiteLLM [Standard Logging Payload is logged for each LLM call](../proxy/logging_spec) | @@ -1465,9 +1465,9 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur ::: -| Property | Details | -|----------|---------| -| Description | Log LLM Input/Output to Azure Blob Storage (Bucket) | +| Property | Details | +| ------------------------------- | --------------------------------------------------------------------------------------------------------------- | +| Description | Log LLM Input/Output to Azure Blob Storage (Bucket) | | Azure Docs on Data Lake Storage | [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction) | @@ -1966,9 +1966,9 @@ This is an Enterprise only feature [Get Started with Enterprise here](https://gi ::: -| Property | Details | -|----------|---------| -| Description | Log LLM Input/Output to a custom API endpoint | +| Property | Details | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Description | Log LLM Input/Output to a custom API endpoint | | Logged Payload | `List[StandardLoggingPayload]` LiteLLM logs a list of [`StandardLoggingPayload` objects](https://docs.litellm.ai/docs/proxy/logging_spec) to your endpoint | @@ -1995,10 +1995,10 @@ litellm_settings: 2. Set Environment Variables for the custom API endpoint -| Environment Variable | Details | Required | -|----------|---------|----------| -| `GENERIC_LOGGER_ENDPOINT` | The endpoint + route we should send callback logs to | Yes | -| `GENERIC_LOGGER_HEADERS` | Optional: Set headers to be sent to the custom API endpoint | No, this is optional | +| Environment Variable | Details | Required | +| ------------------------- | ----------------------------------------------------------- | -------------------- | +| `GENERIC_LOGGER_ENDPOINT` | The endpoint + route we should send callback logs to | Yes | +| `GENERIC_LOGGER_HEADERS` | Optional: Set headers to be sent to the custom API endpoint | No, this is optional | ```shell showLineNumbers title=".env" GENERIC_LOGGER_ENDPOINT="https://webhook-test.com/30343bc33591bc5e6dc44217ceae3e0a" @@ -2428,6 +2428,7 @@ export SENTRY_DSN="your-sentry-dsn" # Optional: Configure Sentry sampling rates export SENTRY_API_SAMPLE_RATE="1.0" # Controls what percentage of errors are sent (default: 1.0 = 100%) export SENTRY_API_TRACE_RATE="1.0" # Controls what percentage of transactions are sampled for performance monitoring (default: 1.0 = 100%) +export SENTRY_ENVIRONMENT="development" # Controls the Sentry Environment (default: production) ``` ```yaml diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 773ff29e371..cff8ced87af 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -1197,7 +1197,7 @@ class Logging(LiteLLMLoggingBaseClass): total_cost=total_cost, tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar, ) - + # Store discount information if provided if original_cost is not None: self.cost_breakdown["original_cost"] = original_cost @@ -1205,7 +1205,7 @@ class Logging(LiteLLMLoggingBaseClass): self.cost_breakdown["discount_percent"] = discount_percent if discount_amount is not None: self.cost_breakdown["discount_amount"] = discount_amount - + def _response_cost_calculator( @@ -3103,7 +3103,7 @@ def _get_masked_values( ( v[: unmasked_length // 2] + "*" * number_of_asterisks - + v[-unmasked_length // 2 :] + + v[-unmasked_length // 2:] ) if ( isinstance(v, str) @@ -3114,7 +3114,7 @@ def _get_masked_values( ( v[: unmasked_length // 2] + "*" * (len(v) - unmasked_length) - + v[-unmasked_length // 2 :] + + v[-unmasked_length // 2:] ) if (isinstance(v, str) and len(v) > unmasked_length) else ("*****" if isinstance(v, str) else v) @@ -3165,6 +3165,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915 event_scrubber=EventScrubber( denylist=SENTRY_DENYLIST, pii_denylist=SENTRY_PII_DENYLIST ), + environment=os.environ.get("SENTRY_ENVIRONMENT", "production"), ) capture_exception = sentry_sdk_instance.capture_exception add_breadcrumb = sentry_sdk_instance.add_breadcrumb @@ -4471,12 +4472,12 @@ def _get_status_fields( ) -> "StandardLoggingPayloadStatusFields": """ Determine status fields based on request status and guardrail information. - + Args: status: Overall request status ("success" or "failure") guardrail_information: Guardrail information from metadata error_str: Error string if any - + Returns: StandardLoggingPayloadStatusFields with llm_api_status and guardrail_status """ @@ -4489,10 +4490,10 @@ def _get_status_fields( "guardrail_failed_to_respond": "guardrail_failed_to_respond", # direct "not_run": "not_run" } - + # Set LLM API status llm_api_status: StandardLoggingPayloadStatus = status - + ######################################################### # Map - guardrail_information.guardrail_status to guardrail_status diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py index 3ab3455d011..2917b156a6f 100644 --- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py +++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py @@ -64,6 +64,80 @@ def test_sentry_sample_rate(): del os.environ["SENTRY_API_SAMPLE_RATE"] +def test_sentry_environment(): + """Test that SENTRY_ENVIRONMENT is properly handled during Sentry initialization""" + existing_environment = os.getenv("SENTRY_ENVIRONMENT") + existing_dsn = os.getenv("SENTRY_DSN") + + # Create mock sentry_sdk module + mock_event_scrubber_instance = MagicMock() + mock_event_scrubber_cls = MagicMock(return_value=mock_event_scrubber_instance) + + mock_scrubber_module = MagicMock() + mock_scrubber_module.EventScrubber = mock_event_scrubber_cls + + mock_sentry_sdk = MagicMock() + mock_sentry_sdk.scrubber = mock_scrubber_module + mock_init = MagicMock() + mock_sentry_sdk.init = mock_init + + # Inject mocks into sys.modules + sys.modules["sentry_sdk"] = mock_sentry_sdk + sys.modules["sentry_sdk.scrubber"] = mock_scrubber_module + + try: + # Set a mock DSN to allow Sentry initialization + os.environ["SENTRY_DSN"] = "https://test@sentry.io/123456" + + # Test with default value (no environment set) + if existing_environment: + del os.environ["SENTRY_ENVIRONMENT"] + + mock_init.reset_mock() + set_callbacks(["sentry"]) + # Check that init was called with default environment "production" + mock_init.assert_called_once() + call_kwargs = mock_init.call_args[1] + assert call_kwargs["environment"] == "production" + + # Test with custom environment value + os.environ["SENTRY_ENVIRONMENT"] = "development" + + mock_init.reset_mock() + set_callbacks(["sentry"]) + # Check that init was called with custom environment "development" + mock_init.assert_called_once() + call_kwargs = mock_init.call_args[1] + assert call_kwargs["environment"] == "development" + + # Test with staging environment + os.environ["SENTRY_ENVIRONMENT"] = "staging" + + mock_init.reset_mock() + set_callbacks(["sentry"]) + # Check that init was called with custom environment "staging" + mock_init.assert_called_once() + call_kwargs = mock_init.call_args[1] + assert call_kwargs["environment"] == "staging" + + except Exception as e: + print(f"Error: {e}") + raise + finally: + # Restore the original environment variables + if existing_environment: + os.environ["SENTRY_ENVIRONMENT"] = existing_environment + else: + if "SENTRY_ENVIRONMENT" in os.environ: + del os.environ["SENTRY_ENVIRONMENT"] + + if existing_dsn: + os.environ["SENTRY_DSN"] = existing_dsn + else: + if "SENTRY_DSN" in os.environ: + del os.environ["SENTRY_DSN"] + + def test_use_custom_pricing_for_model(): from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model