diff --git a/docs/my-website/docs/proxy/prometheus.md b/docs/my-website/docs/proxy/prometheus.md index b1aae1da7c6..dc7030949bd 100644 --- a/docs/my-website/docs/proxy/prometheus.md +++ b/docs/my-website/docs/proxy/prometheus.md @@ -261,7 +261,12 @@ model_list: litellm_settings: callbacks: ["prometheus"] custom_prometheus_metadata_labels: ["metadata.foo", "metadata.bar"] - custom_prometheus_tags: ["prod", "staging", "batch-job"] + custom_prometheus_tags: + - "prod" + - "staging" + - "batch-job" + - "User-Agent: RooCode/*" + - "User-Agent: claude-cli/*" ``` 2. Make a request with tags @@ -297,16 +302,26 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ ``` **How Custom Tags Work:** -- Each configured tag becomes a boolean label in prometheus metrics -- If a tag is present in the request, the label value is `"true"` -- If a tag is not present in the request, the label value is `"false"` +- Each configured tag becomes a boolean label in prometheus metrics +- If a tag matches (exact or wildcard), the label value is `"true"`, otherwise `"false"` - Tag names are sanitized for prometheus compatibility (e.g., `"batch-job"` becomes `"tag_batch_job"`) +- **Wildcard patterns** supported using `*` (e.g., `"User-Agent: RooCode/*"` matches `"User-Agent: RooCode/1.0.0"`) + +**Example with wildcards:** +```yaml +litellm_settings: + callbacks: ["prometheus"] + custom_prometheus_tags: + - "User-Agent: RooCode/*" + - "User-Agent: claude-cli/*" +``` **Use Cases:** - Environment tracking (`prod`, `staging`, `dev`) - Request type classification (`batch-job`, `user-facing`, `background`) - Feature flags (`new-feature`, `beta-users`) - Team or service identification (`team-a`, `service-xyz`) +- User-Agent Tracking - use this to track how much Roo Code, Claude Code, Gemini CLI are used (`User-Agent: RooCode/*`, `User-Agent: claude-cli/*`, `User-Agent: gemini-cli/*`) ## Configuring Metrics and Labels diff --git a/enterprise/litellm_enterprise/integrations/prometheus.py b/enterprise/litellm_enterprise/integrations/prometheus.py index b3bff733e77..a2d781fa1c4 100644 --- a/enterprise/litellm_enterprise/integrations/prometheus.py +++ b/enterprise/litellm_enterprise/integrations/prometheus.py @@ -2293,10 +2293,60 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]: return result +def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: str) -> bool: + """ + Check if any of the request tags matches a wildcard configured pattern + + Args: + tags: List[str] - The request tags + configured_tag: str - The configured tag + + Returns: + bool - True if any of the request tags matches the configured tag, False otherwise + + e.g. + tags = ["User-Agent: curl/7.68.0", "User-Agent: python-requests/2.28.1", "prod"] + configured_tag = "User-Agent: curl/*" + _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True + + configured_tag = "User-Agent: python-requests/*" + _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True + + configured_tag = "gm" + _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # False + """ + import re + + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter + pattern_router = PatternMatchRouter() + regex_pattern = pattern_router._pattern_to_regex(configured_tag) + return any(re.match(pattern=regex_pattern, string=tag) for tag in tags) + + def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]: """ - Get custom labels from tags based on admin configuration + Get custom labels from tags based on admin configuration. + + Supports both exact matches and wildcard patterns: + - Exact match: "prod" matches "prod" exactly + - Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0" + + Reuses PatternMatchRouter for wildcard pattern matching. + + Returns dict of label_name: "true" if the tag matches the configured tag, "false" otherwise + + { + "tag_User-Agent_curl": "true", + "tag_User-Agent_python_requests": "false", + "tag_Environment_prod": "true", + "tag_Environment_dev": "false", + "tag_Service_api_gateway_v2": "true", + "tag_Service_web_app_v1": "false", + } """ + import re + + from litellm.router_utils.pattern_match_deployments import PatternMatchRouter from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name configured_tags = litellm.custom_prometheus_tags @@ -2304,16 +2354,22 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]: return {} result: Dict[str, str] = {} + pattern_router = PatternMatchRouter() - # Map each configured tag to its presence in the request tags for configured_tag in configured_tags: - # Create a safe prometheus label name label_name = _sanitize_prometheus_label_name(f"tag_{configured_tag}") - - # Check if this tag is present in the request tags + + # Check for exact match first (backwards compatibility) if configured_tag in tags: result[label_name] = "true" - else: - result[label_name] = "false" + continue + + # Use PatternMatchRouter for wildcard pattern matching + if "*" in configured_tag and _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag): + result[label_name] = "true" + continue + + # No match found + result[label_name] = "false" return result diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index cfb38299541..936d93c4417 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -1135,6 +1135,119 @@ def test_get_custom_labels_from_tags_no_tags(monkeypatch): } +def test_get_custom_labels_from_tags_wildcard_patterns(monkeypatch): + """Test wildcard pattern matching for custom labels from tags.""" + from litellm_enterprise.integrations.prometheus import get_custom_labels_from_tags + + # Configure tags with wildcard patterns + monkeypatch.setattr( + "litellm.custom_prometheus_tags", + ["User-Agent: curl/*", "User-Agent: python-requests/*", "Environment: prod*", "Service: api-gateway*", "exact-match"] + ) + + # Test tags that should match the wildcard patterns + tags = [ + "User-Agent: curl/7.68.0", + "User-Agent: python-requests/2.28.1", + "Environment: production", + "Service: api-gateway-v2", + "exact-match", + "other-tag" + ] + + result = get_custom_labels_from_tags(tags) + + expected = { + "tag_User_Agent__curl__": "true", # matches "User-Agent: curl/*" + "tag_User_Agent__python_requests__": "true", # matches "User-Agent: python-requests/*" + "tag_Environment__prod_": "true", # matches "Environment: prod*" + "tag_Service__api_gateway_": "true", # matches "Service: api-gateway*" + "tag_exact_match": "true", # exact match + } + + assert result == expected + + +def test_get_custom_labels_from_tags_wildcard_no_matches(monkeypatch): + """Test wildcard patterns that don't match any tags.""" + from litellm_enterprise.integrations.prometheus import get_custom_labels_from_tags + + # Configure tags with wildcard patterns + monkeypatch.setattr( + "litellm.custom_prometheus_tags", + ["User-Agent: firefox/*", "Environment: dev*", "Service: web-app*"] + ) + + # Test tags that should NOT match the wildcard patterns + tags = [ + "User-Agent: curl/7.68.0", # doesn't match "User-Agent: firefox/*" + "Environment: production", # doesn't match "Environment: dev*" + "Service: api-gateway-v2", # doesn't match "Service: web-app*" + "other-tag" + ] + + result = get_custom_labels_from_tags(tags) + + expected = { + "tag_User_Agent__firefox__": "false", # no match for "User-Agent: firefox/*" + "tag_Environment__dev_": "false", # no match for "Environment: dev*" + "tag_Service__web_app_": "false", # no match for "Service: web-app*" + } + + assert result == expected + + +def test_tag_matches_wildcard_configured_pattern(): + """Test the helper function for wildcard pattern matching.""" + from litellm_enterprise.integrations.prometheus import ( + _tag_matches_wildcard_configured_pattern, + ) + + # Test cases that should match + assert _tag_matches_wildcard_configured_pattern( + tags=["User-Agent: curl/7.68.0", "prod", "other"], + configured_tag="User-Agent: curl/*" + ) is True + + assert _tag_matches_wildcard_configured_pattern( + tags=["User-Agent: python-requests/2.28.1", "test"], + configured_tag="User-Agent: python-requests/*" + ) is True + + assert _tag_matches_wildcard_configured_pattern( + tags=["Environment: production", "debug"], + configured_tag="Environment: prod*" + ) is True + + # Test exact match (no wildcard) + assert _tag_matches_wildcard_configured_pattern( + tags=["prod", "test"], + configured_tag="prod" + ) is True + + # Test cases that should NOT match + assert _tag_matches_wildcard_configured_pattern( + tags=["User-Agent: firefox/98.0", "prod"], + configured_tag="User-Agent: curl/*" + ) is False + + assert _tag_matches_wildcard_configured_pattern( + tags=["Environment: development", "test"], + configured_tag="Environment: prod*" + ) is False + + assert _tag_matches_wildcard_configured_pattern( + tags=["staging", "test"], + configured_tag="prod" + ) is False + + # Test with empty tags + assert _tag_matches_wildcard_configured_pattern( + tags=[], + configured_tag="User-Agent: curl/*" + ) is False + + @pytest.mark.asyncio(scope="session") async def test_initialize_remaining_budget_metrics(prometheus_logger): """