From 8216ec43896751fe85184e6119fa24a45e451757 Mon Sep 17 00:00:00 2001
From: Ryan Crabbe <rcrabbe@berkeley.edu>
Date: Mon, 26 Jan 2026 12:57:39 -0800
Subject: [PATCH] perf: add LRU cache to normalize_request_route

Add @lru_cache(maxsize=256) to eliminate redundant regex work for
repeated routes. Reduces time from 1.04s to ~0s for 6,006 calls.
---
 litellm/proxy/auth/auth_utils.py | 84 ++++++++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py
index 1a7f05716b3..ab40ee1ce91 100644
--- a/litellm/proxy/auth/auth_utils.py
+++ b/litellm/proxy/auth/auth_utils.py
@@ -1,6 +1,7 @@
 import os
 import re
 import sys
+from functools import lru_cache
 from typing import Any, List, Optional, Tuple
 
 from fastapi import HTTPException, Request, status
@@ -311,6 +312,89 @@ def get_request_route(request: Request) -> str:
         return request.url.path
 
 
+@lru_cache(maxsize=256)
+def normalize_request_route(route: str) -> str:
+    """
+    Normalize request routes by replacing dynamic path parameters with placeholders.
+
+    This prevents high cardinality in Prometheus metrics by collapsing routes like:
+    - /v1/responses/1234567890 -> /v1/responses/{response_id}
+    - /v1/threads/thread_123 -> /v1/threads/{thread_id}
+    
+    Args:
+        route: The request route path
+        
+    Returns:
+        Normalized route with dynamic parameters replaced by placeholders
+        
+    Examples:
+        >>> normalize_request_route("/v1/responses/abc123")
+        '/v1/responses/{response_id}'
+        >>> normalize_request_route("/v1/responses/abc123/cancel")
+        '/v1/responses/{response_id}/cancel'
+        >>> normalize_request_route("/chat/completions")
+        '/chat/completions'
+    """
+    # Define patterns for routes with dynamic IDs
+    # Format: (regex_pattern, replacement_template)
+    patterns = [
+        # Responses API - must come before generic patterns
+        (r'^(/(?:openai/)?v1/responses)/([^/]+)(/input_items)$', r'\1/{response_id}\3'),
+        (r'^(/(?:openai/)?v1/responses)/([^/]+)(/cancel)$', r'\1/{response_id}\3'),
+        (r'^(/(?:openai/)?v1/responses)/([^/]+)$', r'\1/{response_id}'),
+        (r'^(/responses)/([^/]+)(/input_items)$', r'\1/{response_id}\3'),
+        (r'^(/responses)/([^/]+)(/cancel)$', r'\1/{response_id}\3'),
+        (r'^(/responses)/([^/]+)$', r'\1/{response_id}'),
+        
+        # Threads API
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)/([^/]+)$', r'\1/{thread_id}\3/{run_id}\5/{step_id}'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/steps)$', r'\1/{thread_id}\3/{run_id}\5'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/cancel)$', r'\1/{thread_id}\3/{run_id}\5'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)(/submit_tool_outputs)$', r'\1/{thread_id}\3/{run_id}\5'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)/([^/]+)$', r'\1/{thread_id}\3/{run_id}'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/runs)$', r'\1/{thread_id}\3'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/messages)/([^/]+)$', r'\1/{thread_id}\3/{message_id}'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)(/messages)$', r'\1/{thread_id}\3'),
+        (r'^(/(?:openai/)?v1/threads)/([^/]+)$', r'\1/{thread_id}'),
+        
+        # Vector Stores API
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/files)/([^/]+)$', r'\1/{vector_store_id}\3/{file_id}'),
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/files)$', r'\1/{vector_store_id}\3'),
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/file_batches)/([^/]+)$', r'\1/{vector_store_id}\3/{batch_id}'),
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)(/file_batches)$', r'\1/{vector_store_id}\3'),
+        (r'^(/(?:openai/)?v1/vector_stores)/([^/]+)$', r'\1/{vector_store_id}'),
+        
+        # Assistants API
+        (r'^(/(?:openai/)?v1/assistants)/([^/]+)$', r'\1/{assistant_id}'),
+        
+        # Files API
+        (r'^(/(?:openai/)?v1/files)/([^/]+)(/content)$', r'\1/{file_id}\3'),
+        (r'^(/(?:openai/)?v1/files)/([^/]+)$', r'\1/{file_id}'),
+        
+        # Batches API
+        (r'^(/(?:openai/)?v1/batches)/([^/]+)(/cancel)$', r'\1/{batch_id}\3'),
+        (r'^(/(?:openai/)?v1/batches)/([^/]+)$', r'\1/{batch_id}'),
+        
+        # Fine-tuning API
+        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/events)$', r'\1/{fine_tuning_job_id}\3'),
+        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/cancel)$', r'\1/{fine_tuning_job_id}\3'),
+        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)(/checkpoints)$', r'\1/{fine_tuning_job_id}\3'),
+        (r'^(/(?:openai/)?v1/fine_tuning/jobs)/([^/]+)$', r'\1/{fine_tuning_job_id}'),
+        
+        # Models API
+        (r'^(/(?:openai/)?v1/models)/([^/]+)$', r'\1/{model}'),
+    ]
+    
+    # Apply patterns in order
+    for pattern, replacement in patterns:
+        normalized = re.sub(pattern, replacement, route)
+        if normalized != route:
+            return normalized
+    
+    # Return original route if no pattern matched
+    return route
+
+
 async def check_if_request_size_is_safe(request: Request) -> bool:
     """
     Enterprise Only: