[Feat] - When using custom tags on prometheus allow using wildcard patterns (#13316)

* _tag_matches_wildcard_configured_pattern

* test_get_custom_labels_from_tags_wildcard_patterns

* docs Custom Tags

* docs how custom tags work

* fix

docs/my-website/docs/proxy/prometheus.md

@@ -261,7 +261,12 @@ model_list:
 litellm_settings:
   callbacks: ["prometheus"]
   custom_prometheus_metadata_labels: ["metadata.foo", "metadata.bar"]
-  custom_prometheus_tags: ["prod", "staging", "batch-job"]
+  custom_prometheus_tags: 
+    - "prod"
+    - "staging"
+    - "batch-job"
+    - "User-Agent: RooCode/*"
+    - "User-Agent: claude-cli/*"
 ```
 
 2. Make a request with tags
@@ -297,16 +302,26 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 ```
 
 **How Custom Tags Work:**
-- Each configured tag becomes a boolean label in prometheus metrics
-- If a tag is present in the request, the label value is `"true"`
-- If a tag is not present in the request, the label value is `"false"`
+- Each configured tag becomes a boolean label in prometheus metrics  
+- If a tag matches (exact or wildcard), the label value is `"true"`, otherwise `"false"`
 - Tag names are sanitized for prometheus compatibility (e.g., `"batch-job"` becomes `"tag_batch_job"`)
+- **Wildcard patterns** supported using `*` (e.g., `"User-Agent: RooCode/*"` matches `"User-Agent: RooCode/1.0.0"`)
+
+**Example with wildcards:**
+```yaml
+litellm_settings:
+  callbacks: ["prometheus"]
+  custom_prometheus_tags:
+    - "User-Agent: RooCode/*"
+    - "User-Agent: claude-cli/*"
+``` 
 
 **Use Cases:**
 - Environment tracking (`prod`, `staging`, `dev`)
 - Request type classification (`batch-job`, `user-facing`, `background`)
 - Feature flags (`new-feature`, `beta-users`)
 - Team or service identification (`team-a`, `service-xyz`)
+- User-Agent Tracking - use this to track how much Roo Code, Claude Code, Gemini CLI are used (`User-Agent: RooCode/*`, `User-Agent: claude-cli/*`, `User-Agent: gemini-cli/*`)
 
 
 ## Configuring Metrics and Labels

enterprise/litellm_enterprise/integrations/prometheus.py

@@ -2293,10 +2293,60 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
     return result
 
 
+def _tag_matches_wildcard_configured_pattern(tags: List[str], configured_tag: str) -> bool:
+    """
+    Check if any of the request tags matches a wildcard configured pattern
+
+    Args:
+        tags: List[str] - The request tags
+        configured_tag: str - The configured tag
+
+    Returns:
+        bool - True if any of the request tags matches the configured tag, False otherwise
+
+    e.g.
+    tags = ["User-Agent: curl/7.68.0", "User-Agent: python-requests/2.28.1", "prod"]
+    configured_tag = "User-Agent: curl/*"
+    _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True
+
+    configured_tag = "User-Agent: python-requests/*"
+    _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # True
+
+    configured_tag = "gm"
+    _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag) # False
+    """
+    import re
+
+    from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
+    pattern_router = PatternMatchRouter()
+    regex_pattern = pattern_router._pattern_to_regex(configured_tag)
+    return any(re.match(pattern=regex_pattern, string=tag) for tag in tags)
+
+
 def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
     """
-    Get custom labels from tags based on admin configuration
+    Get custom labels from tags based on admin configuration.
+    
+    Supports both exact matches and wildcard patterns:
+    - Exact match: "prod" matches "prod" exactly
+    - Wildcard pattern: "User-Agent: curl/*" matches "User-Agent: curl/7.68.0" 
+    
+    Reuses PatternMatchRouter for wildcard pattern matching.
+
+    Returns dict of label_name: "true" if the tag matches the configured tag, "false" otherwise
+
+    {
+        "tag_User-Agent_curl": "true",
+        "tag_User-Agent_python_requests": "false",
+        "tag_Environment_prod": "true",
+        "tag_Environment_dev": "false",
+        "tag_Service_api_gateway_v2": "true",
+        "tag_Service_web_app_v1": "false",
+    }
     """
+    import re
+
+    from litellm.router_utils.pattern_match_deployments import PatternMatchRouter
     from litellm.types.integrations.prometheus import _sanitize_prometheus_label_name
 
     configured_tags = litellm.custom_prometheus_tags
@@ -2304,16 +2354,22 @@ def get_custom_labels_from_tags(tags: List[str]) -> Dict[str, str]:
         return {}
 
     result: Dict[str, str] = {}
+    pattern_router = PatternMatchRouter()
 
-    # Map each configured tag to its presence in the request tags
     for configured_tag in configured_tags:
-        # Create a safe prometheus label name
         label_name = _sanitize_prometheus_label_name(f"tag_{configured_tag}")
-
-        # Check if this tag is present in the request tags
+        
+        # Check for exact match first (backwards compatibility)
         if configured_tag in tags:
             result[label_name] = "true"
-        else:
-            result[label_name] = "false"
+            continue
+            
+        # Use PatternMatchRouter for wildcard pattern matching
+        if "*" in configured_tag and _tag_matches_wildcard_configured_pattern(tags=tags, configured_tag=configured_tag):
+            result[label_name] = "true"
+            continue
+        
+        # No match found
+        result[label_name] = "false"
 
     return result

tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py

@@ -1135,6 +1135,119 @@ def test_get_custom_labels_from_tags_no_tags(monkeypatch):
     }
 
 
+def test_get_custom_labels_from_tags_wildcard_patterns(monkeypatch):
+    """Test wildcard pattern matching for custom labels from tags."""
+    from litellm_enterprise.integrations.prometheus import get_custom_labels_from_tags
+
+    # Configure tags with wildcard patterns
+    monkeypatch.setattr(
+        "litellm.custom_prometheus_tags", 
+        ["User-Agent: curl/*", "User-Agent: python-requests/*", "Environment: prod*", "Service: api-gateway*", "exact-match"]
+    )
+    
+    # Test tags that should match the wildcard patterns
+    tags = [
+        "User-Agent: curl/7.68.0", 
+        "User-Agent: python-requests/2.28.1", 
+        "Environment: production",
+        "Service: api-gateway-v2",
+        "exact-match",
+        "other-tag"
+    ]
+    
+    result = get_custom_labels_from_tags(tags)
+    
+    expected = {
+        "tag_User_Agent__curl__": "true",  # matches "User-Agent: curl/*"
+        "tag_User_Agent__python_requests__": "true",  # matches "User-Agent: python-requests/*"
+        "tag_Environment__prod_": "true",  # matches "Environment: prod*"
+        "tag_Service__api_gateway_": "true",  # matches "Service: api-gateway*"
+        "tag_exact_match": "true",  # exact match
+    }
+    
+    assert result == expected
+
+
+def test_get_custom_labels_from_tags_wildcard_no_matches(monkeypatch):
+    """Test wildcard patterns that don't match any tags."""
+    from litellm_enterprise.integrations.prometheus import get_custom_labels_from_tags
+
+    # Configure tags with wildcard patterns
+    monkeypatch.setattr(
+        "litellm.custom_prometheus_tags", 
+        ["User-Agent: firefox/*", "Environment: dev*", "Service: web-app*"]
+    )
+    
+    # Test tags that should NOT match the wildcard patterns
+    tags = [
+        "User-Agent: curl/7.68.0",  # doesn't match "User-Agent: firefox/*"
+        "Environment: production",  # doesn't match "Environment: dev*" 
+        "Service: api-gateway-v2",  # doesn't match "Service: web-app*"
+        "other-tag"
+    ]
+    
+    result = get_custom_labels_from_tags(tags)
+    
+    expected = {
+        "tag_User_Agent__firefox__": "false",  # no match for "User-Agent: firefox/*"
+        "tag_Environment__dev_": "false",  # no match for "Environment: dev*"
+        "tag_Service__web_app_": "false",  # no match for "Service: web-app*"
+    }
+    
+    assert result == expected
+
+
+def test_tag_matches_wildcard_configured_pattern():
+    """Test the helper function for wildcard pattern matching."""
+    from litellm_enterprise.integrations.prometheus import (
+        _tag_matches_wildcard_configured_pattern,
+    )
+
+    # Test cases that should match
+    assert _tag_matches_wildcard_configured_pattern(
+        tags=["User-Agent: curl/7.68.0", "prod", "other"], 
+        configured_tag="User-Agent: curl/*"
+    ) is True
+    
+    assert _tag_matches_wildcard_configured_pattern(
+        tags=["User-Agent: python-requests/2.28.1", "test"], 
+        configured_tag="User-Agent: python-requests/*"
+    ) is True
+    
+    assert _tag_matches_wildcard_configured_pattern(
+        tags=["Environment: production", "debug"], 
+        configured_tag="Environment: prod*"
+    ) is True
+    
+    # Test exact match (no wildcard)
+    assert _tag_matches_wildcard_configured_pattern(
+        tags=["prod", "test"], 
+        configured_tag="prod"
+    ) is True
+    
+    # Test cases that should NOT match
+    assert _tag_matches_wildcard_configured_pattern(
+        tags=["User-Agent: firefox/98.0", "prod"], 
+        configured_tag="User-Agent: curl/*"
+    ) is False
+    
+    assert _tag_matches_wildcard_configured_pattern(
+        tags=["Environment: development", "test"], 
+        configured_tag="Environment: prod*"
+    ) is False
+    
+    assert _tag_matches_wildcard_configured_pattern(
+        tags=["staging", "test"], 
+        configured_tag="prod"
+    ) is False
+    
+    # Test with empty tags
+    assert _tag_matches_wildcard_configured_pattern(
+        tags=[], 
+        configured_tag="User-Agent: curl/*"
+    ) is False
+
+
 @pytest.mark.asyncio(scope="session")
 async def test_initialize_remaining_budget_metrics(prometheus_logger):
     """