mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-10 03:28:53 +00:00
[Feat] Add SENTRY_ENVIRONMENT configuration for Sentry integration (#15760)
* [Feat] Add SENTRY_ENVIRONMENT configuration for Sentry integration and corresponding tests * [Refactor] Enhance test_sentry_environment by mocking sentry_sdk and improving environment handling * [Fix] Update default SENTRY_ENVIRONMENT to 'production' and enhance test for Sentry integration * [Fix] Update test_sentry_environment to verify correct handling of SENTRY_ENVIRONMENT values * [Fix] Update test_sentry_environment to assert correct handling of production environment
This commit is contained in:
Thomas Mildner
GitHub
parent
1fcadd6c05
commit
1cfc4624c3
4 changed files with 119 additions and 37 deletions
|
|
@ -61,6 +61,12 @@ print(response)
|
|||
|
||||
These options are useful for high-volume applications where sampling a subset of errors and transactions provides sufficient visibility while managing costs.
|
||||
|
||||
#### Sentry Environment
|
||||
- **SENTRY_ENVIRONMENT**: Specifies the environment name for your Sentry events (e.g., "production", "staging", "development")
|
||||
- Helps organize and filter errors by deployment environment in Sentry dashboard
|
||||
- Example: `os.environ["SENTRY_ENVIRONMENT"] = "staging"`
|
||||
- If not set, Sentry will use 'production' as the default environment
|
||||
|
||||
## Redacting Messages, Response Content from Sentry Logging
|
||||
|
||||
Set `litellm.turn_off_message_logging=True` This will prevent the messages and responses from being logged to sentry, but request metadata will still be logged.
|
||||
|
|
|
|||
|
|
@ -602,15 +602,15 @@ print(response)
|
|||
|
||||
Use this if you want to control which LiteLLM-specific fields are logged as tags by the LiteLLM proxy. By default LiteLLM Proxy logs no LiteLLM-specific fields
|
||||
|
||||
| LiteLLM specific field | Description | Example Value |
|
||||
|---------------------------|-----------------------------------------------------------------------------------------|------------------------------------------------|
|
||||
| `cache_hit` | Indicates whether a cache hit occurred (True) or not (False) | `true`, `false` |
|
||||
| `cache_key` | The Cache key used for this request | `d2b758c****` |
|
||||
| `proxy_base_url` | The base URL for the proxy server, the value of env var `PROXY_BASE_URL` on your server | `https://proxy.example.com` |
|
||||
| `user_api_key_alias` | An alias for the LiteLLM Virtual Key. | `prod-app1` |
|
||||
| `user_api_key_user_id` | The unique ID associated with a user's API key. | `user_123`, `user_456` |
|
||||
| `user_api_key_user_email` | The email associated with a user's API key. | `user@example.com`, `admin@example.com` |
|
||||
| `user_api_key_team_alias` | An alias for a team associated with an API key. | `team_alpha`, `dev_team` |
|
||||
| LiteLLM specific field | Description | Example Value |
|
||||
| ------------------------- | --------------------------------------------------------------------------------------- | --------------------------------------- |
|
||||
| `cache_hit` | Indicates whether a cache hit occurred (True) or not (False) | `true`, `false` |
|
||||
| `cache_key` | The Cache key used for this request | `d2b758c****` |
|
||||
| `proxy_base_url` | The base URL for the proxy server, the value of env var `PROXY_BASE_URL` on your server | `https://proxy.example.com` |
|
||||
| `user_api_key_alias` | An alias for the LiteLLM Virtual Key. | `prod-app1` |
|
||||
| `user_api_key_user_id` | The unique ID associated with a user's API key. | `user_123`, `user_456` |
|
||||
| `user_api_key_user_email` | The email associated with a user's API key. | `user@example.com`, `admin@example.com` |
|
||||
| `user_api_key_team_alias` | An alias for a team associated with an API key. | `team_alpha`, `dev_team` |
|
||||
|
||||
|
||||
**Usage**
|
||||
|
|
@ -1111,10 +1111,10 @@ Log LLM Logs to [Google Cloud Storage Buckets](https://cloud.google.com/storage?
|
|||
:::
|
||||
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Description | Log LLM Input/Output to cloud storage buckets |
|
||||
| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) |
|
||||
| Property | Details |
|
||||
| ---------------------------- | -------------------------------------------------------------- |
|
||||
| Description | Log LLM Input/Output to cloud storage buckets |
|
||||
| Load Test Benchmarks | [Benchmarks](https://docs.litellm.ai/docs/benchmarks) |
|
||||
| Google Docs on Cloud Storage | [Google Cloud Storage](https://cloud.google.com/storage?hl=en) |
|
||||
|
||||
|
||||
|
|
@ -1196,8 +1196,8 @@ Log LLM Logs/SpendLogs to [Google Cloud Storage PubSub Topic](https://cloud.goog
|
|||
:::
|
||||
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Property | Details |
|
||||
| ----------- | ------------------------------------------------------------------ |
|
||||
| Description | Log LiteLLM `SpendLogs Table` to Google Cloud Storage PubSub Topic |
|
||||
|
||||
When to use `gcs_pubsub`?
|
||||
|
|
@ -1388,10 +1388,10 @@ On s3 bucket, you will see the object key as `my-test-path/my-team-alias/...`
|
|||
## AWS SQS
|
||||
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Description | Log LLM Input/Output to AWS SQS Queue |
|
||||
| AWS Docs on SQS | [AWS SQS](https://aws.amazon.com/sqs/) |
|
||||
| Property | Details |
|
||||
| -------------------- | ------------------------------------------------------------------------------------- |
|
||||
| Description | Log LLM Input/Output to AWS SQS Queue |
|
||||
| AWS Docs on SQS | [AWS SQS](https://aws.amazon.com/sqs/) |
|
||||
| Fields Logged to SQS | LiteLLM [Standard Logging Payload is logged for each LLM call](../proxy/logging_spec) |
|
||||
|
||||
|
||||
|
|
@ -1465,9 +1465,9 @@ Log LLM Logs to [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azur
|
|||
:::
|
||||
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Description | Log LLM Input/Output to Azure Blob Storage (Bucket) |
|
||||
| Property | Details |
|
||||
| ------------------------------- | --------------------------------------------------------------------------------------------------------------- |
|
||||
| Description | Log LLM Input/Output to Azure Blob Storage (Bucket) |
|
||||
| Azure Docs on Data Lake Storage | [Azure Data Lake Storage](https://learn.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction) |
|
||||
|
||||
|
||||
|
|
@ -1966,9 +1966,9 @@ This is an Enterprise only feature [Get Started with Enterprise here](https://gi
|
|||
|
||||
:::
|
||||
|
||||
| Property | Details |
|
||||
|----------|---------|
|
||||
| Description | Log LLM Input/Output to a custom API endpoint |
|
||||
| Property | Details |
|
||||
| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| Description | Log LLM Input/Output to a custom API endpoint |
|
||||
| Logged Payload | `List[StandardLoggingPayload]` LiteLLM logs a list of [`StandardLoggingPayload` objects](https://docs.litellm.ai/docs/proxy/logging_spec) to your endpoint |
|
||||
|
||||
|
||||
|
|
@ -1995,10 +1995,10 @@ litellm_settings:
|
|||
|
||||
2. Set Environment Variables for the custom API endpoint
|
||||
|
||||
| Environment Variable | Details | Required |
|
||||
|----------|---------|----------|
|
||||
| `GENERIC_LOGGER_ENDPOINT` | The endpoint + route we should send callback logs to | Yes |
|
||||
| `GENERIC_LOGGER_HEADERS` | Optional: Set headers to be sent to the custom API endpoint | No, this is optional |
|
||||
| Environment Variable | Details | Required |
|
||||
| ------------------------- | ----------------------------------------------------------- | -------------------- |
|
||||
| `GENERIC_LOGGER_ENDPOINT` | The endpoint + route we should send callback logs to | Yes |
|
||||
| `GENERIC_LOGGER_HEADERS` | Optional: Set headers to be sent to the custom API endpoint | No, this is optional |
|
||||
|
||||
```shell showLineNumbers title=".env"
|
||||
GENERIC_LOGGER_ENDPOINT="https://webhook-test.com/30343bc33591bc5e6dc44217ceae3e0a"
|
||||
|
|
@ -2428,6 +2428,7 @@ export SENTRY_DSN="your-sentry-dsn"
|
|||
# Optional: Configure Sentry sampling rates
|
||||
export SENTRY_API_SAMPLE_RATE="1.0" # Controls what percentage of errors are sent (default: 1.0 = 100%)
|
||||
export SENTRY_API_TRACE_RATE="1.0" # Controls what percentage of transactions are sampled for performance monitoring (default: 1.0 = 100%)
|
||||
export SENTRY_ENVIRONMENT="development" # Controls the Sentry Environment (default: production)
|
||||
```
|
||||
|
||||
```yaml
|
||||
|
|
|
|||
|
|
@ -1197,7 +1197,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
total_cost=total_cost,
|
||||
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
|
||||
)
|
||||
|
||||
|
||||
# Store discount information if provided
|
||||
if original_cost is not None:
|
||||
self.cost_breakdown["original_cost"] = original_cost
|
||||
|
|
@ -1205,7 +1205,7 @@ class Logging(LiteLLMLoggingBaseClass):
|
|||
self.cost_breakdown["discount_percent"] = discount_percent
|
||||
if discount_amount is not None:
|
||||
self.cost_breakdown["discount_amount"] = discount_amount
|
||||
|
||||
|
||||
|
||||
|
||||
def _response_cost_calculator(
|
||||
|
|
@ -3103,7 +3103,7 @@ def _get_masked_values(
|
|||
(
|
||||
v[: unmasked_length // 2]
|
||||
+ "*" * number_of_asterisks
|
||||
+ v[-unmasked_length // 2 :]
|
||||
+ v[-unmasked_length // 2:]
|
||||
)
|
||||
if (
|
||||
isinstance(v, str)
|
||||
|
|
@ -3114,7 +3114,7 @@ def _get_masked_values(
|
|||
(
|
||||
v[: unmasked_length // 2]
|
||||
+ "*" * (len(v) - unmasked_length)
|
||||
+ v[-unmasked_length // 2 :]
|
||||
+ v[-unmasked_length // 2:]
|
||||
)
|
||||
if (isinstance(v, str) and len(v) > unmasked_length)
|
||||
else ("*****" if isinstance(v, str) else v)
|
||||
|
|
@ -3165,6 +3165,7 @@ def set_callbacks(callback_list, function_id=None): # noqa: PLR0915
|
|||
event_scrubber=EventScrubber(
|
||||
denylist=SENTRY_DENYLIST, pii_denylist=SENTRY_PII_DENYLIST
|
||||
),
|
||||
environment=os.environ.get("SENTRY_ENVIRONMENT", "production"),
|
||||
)
|
||||
capture_exception = sentry_sdk_instance.capture_exception
|
||||
add_breadcrumb = sentry_sdk_instance.add_breadcrumb
|
||||
|
|
@ -4471,12 +4472,12 @@ def _get_status_fields(
|
|||
) -> "StandardLoggingPayloadStatusFields":
|
||||
"""
|
||||
Determine status fields based on request status and guardrail information.
|
||||
|
||||
|
||||
Args:
|
||||
status: Overall request status ("success" or "failure")
|
||||
guardrail_information: Guardrail information from metadata
|
||||
error_str: Error string if any
|
||||
|
||||
|
||||
Returns:
|
||||
StandardLoggingPayloadStatusFields with llm_api_status and guardrail_status
|
||||
"""
|
||||
|
|
@ -4489,10 +4490,10 @@ def _get_status_fields(
|
|||
"guardrail_failed_to_respond": "guardrail_failed_to_respond", # direct
|
||||
"not_run": "not_run"
|
||||
}
|
||||
|
||||
|
||||
# Set LLM API status
|
||||
llm_api_status: StandardLoggingPayloadStatus = status
|
||||
|
||||
|
||||
|
||||
#########################################################
|
||||
# Map - guardrail_information.guardrail_status to guardrail_status
|
||||
|
|
|
|||
|
|
@ -64,6 +64,80 @@ def test_sentry_sample_rate():
|
|||
del os.environ["SENTRY_API_SAMPLE_RATE"]
|
||||
|
||||
|
||||
def test_sentry_environment():
|
||||
"""Test that SENTRY_ENVIRONMENT is properly handled during Sentry initialization"""
|
||||
existing_environment = os.getenv("SENTRY_ENVIRONMENT")
|
||||
existing_dsn = os.getenv("SENTRY_DSN")
|
||||
|
||||
# Create mock sentry_sdk module
|
||||
mock_event_scrubber_instance = MagicMock()
|
||||
mock_event_scrubber_cls = MagicMock(return_value=mock_event_scrubber_instance)
|
||||
|
||||
mock_scrubber_module = MagicMock()
|
||||
mock_scrubber_module.EventScrubber = mock_event_scrubber_cls
|
||||
|
||||
mock_sentry_sdk = MagicMock()
|
||||
mock_sentry_sdk.scrubber = mock_scrubber_module
|
||||
mock_init = MagicMock()
|
||||
mock_sentry_sdk.init = mock_init
|
||||
|
||||
# Inject mocks into sys.modules
|
||||
sys.modules["sentry_sdk"] = mock_sentry_sdk
|
||||
sys.modules["sentry_sdk.scrubber"] = mock_scrubber_module
|
||||
|
||||
try:
|
||||
# Set a mock DSN to allow Sentry initialization
|
||||
os.environ["SENTRY_DSN"] = "https://test@sentry.io/123456"
|
||||
|
||||
# Test with default value (no environment set)
|
||||
if existing_environment:
|
||||
del os.environ["SENTRY_ENVIRONMENT"]
|
||||
|
||||
mock_init.reset_mock()
|
||||
set_callbacks(["sentry"])
|
||||
# Check that init was called with default environment "production"
|
||||
mock_init.assert_called_once()
|
||||
call_kwargs = mock_init.call_args[1]
|
||||
assert call_kwargs["environment"] == "production"
|
||||
|
||||
# Test with custom environment value
|
||||
os.environ["SENTRY_ENVIRONMENT"] = "development"
|
||||
|
||||
mock_init.reset_mock()
|
||||
set_callbacks(["sentry"])
|
||||
# Check that init was called with custom environment "development"
|
||||
mock_init.assert_called_once()
|
||||
call_kwargs = mock_init.call_args[1]
|
||||
assert call_kwargs["environment"] == "development"
|
||||
|
||||
# Test with staging environment
|
||||
os.environ["SENTRY_ENVIRONMENT"] = "staging"
|
||||
|
||||
mock_init.reset_mock()
|
||||
set_callbacks(["sentry"])
|
||||
# Check that init was called with custom environment "staging"
|
||||
mock_init.assert_called_once()
|
||||
call_kwargs = mock_init.call_args[1]
|
||||
assert call_kwargs["environment"] == "staging"
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
raise
|
||||
finally:
|
||||
# Restore the original environment variables
|
||||
if existing_environment:
|
||||
os.environ["SENTRY_ENVIRONMENT"] = existing_environment
|
||||
else:
|
||||
if "SENTRY_ENVIRONMENT" in os.environ:
|
||||
del os.environ["SENTRY_ENVIRONMENT"]
|
||||
|
||||
if existing_dsn:
|
||||
os.environ["SENTRY_DSN"] = existing_dsn
|
||||
else:
|
||||
if "SENTRY_DSN" in os.environ:
|
||||
del os.environ["SENTRY_DSN"]
|
||||
|
||||
|
||||
def test_use_custom_pricing_for_model():
|
||||
from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue