From 36c9066372d6412cf98d372f86a4016d83456e77 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Wed, 15 Oct 2025 15:12:40 -0700 Subject: [PATCH 1/9] perf(router): optimize model lookups with O(1) index maps and standardize timing - Use model_id_to_deployment_index_map and model_name_to_deployment_indices for O(1) lookups in get_model_info, get_deployment_by_model_group_name, and get_model_ids --- litellm/router.py | 54 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5972b06f01a..82c9d28bb9c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -5519,9 +5519,15 @@ class Router: Returns -> Deployment or None Raise Exception -> if model found in invalid format + + Optimized with O(1) index lookup instead of O(n) linear scan. """ - for model in self.model_list: - if model["model_name"] == model_group_name: + # O(1) lookup in model_name index + if model_group_name in self.model_name_to_deployment_indices: + indices = self.model_name_to_deployment_indices[model_group_name] + if indices: + # Return first deployment for this model_name + model = self.model_list[indices[0]] if isinstance(model, dict): return Deployment(**model) elif isinstance(model, Deployment): @@ -5631,11 +5637,13 @@ class Router: Returns - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info'] - None: could not find deployment in list + + Optimized with O(1) index lookup instead of O(n) linear scan. """ - for model in self.model_list: - if "model_info" in model and "id" in model["model_info"]: - if id == model["model_info"]["id"]: - return model + # O(1) lookup via model_id_to_deployment_index_map + if id in self.model_id_to_deployment_index_map: + idx = self.model_id_to_deployment_index_map[id] + return self.model_list[idx] return None def get_model_group(self, id: str) -> Optional[List]: @@ -6169,17 +6177,33 @@ class Router: if 'model_name' is none, returns all. Returns list of model id's. + + Optimized with O(1) or O(k) index lookup when model_name provided, + instead of O(n) linear scan. """ ids = [] - for model in self.model_list: - if "model_info" in model and "id" in model["model_info"]: - id = model["model_info"]["id"] - if exclude_team_models and model["model_info"].get("team_id"): - continue - if model_name is not None and model["model_name"] == model_name: - ids.append(id) - elif model_name is None: - ids.append(id) + + if model_name is not None: + # O(1) lookup in model_name index, then O(k) iteration where k = deployments for this model_name + if model_name in self.model_name_to_deployment_indices: + indices = self.model_name_to_deployment_indices[model_name] + for idx in indices: + model = self.model_list[idx] + if "model_info" in model and "id" in model["model_info"]: + if exclude_team_models and model["model_info"].get("team_id"): + continue + ids.append(model["model_info"]["id"]) + else: + # When model_name is None, return all model IDs + # Use the index map keys for O(n) where n = total deployments + for model_id in self.model_id_to_deployment_index_map.keys(): + idx = self.model_id_to_deployment_index_map[model_id] + model = self.model_list[idx] + if "model_info" in model and "id" in model["model_info"]: + if exclude_team_models and model["model_info"].get("team_id"): + continue + ids.append(model_id) + return ids def has_model_id(self, candidate_id: str) -> bool: From c98a30a4b64b2198f08710101d0f69d420b2e070 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Wed, 15 Oct 2025 15:44:09 -0700 Subject: [PATCH 2/9] perf(router): optimize string concatenation in hash generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace string concatenation in loop with list append + join pattern. This improves time complexity from O(n²) to O(n) and avoids creating many temporary string objects during hash ID generation. --- litellm/router.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5972b06f01a..f8ce4b23605 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -4915,22 +4915,25 @@ class Router: - hash - use hash as id """ - concat_str = model_group + # Optimized: Use list and join instead of string concatenation in loop + # This avoids creating many temporary string objects (O(n) vs O(n²) complexity) + parts = [model_group] for k, v in litellm_params.items(): if isinstance(k, str): - concat_str += k + parts.append(k) elif isinstance(k, dict): - concat_str += json.dumps(k) + parts.append(json.dumps(k)) else: - concat_str += str(k) + parts.append(str(k)) if isinstance(v, str): - concat_str += v + parts.append(v) elif isinstance(v, dict): - concat_str += json.dumps(v) + parts.append(json.dumps(v)) else: - concat_str += str(v) + parts.append(str(v)) + concat_str = "".join(parts) hash_object = hashlib.sha256(concat_str.encode()) return hash_object.hexdigest() From 97ed3d01ec359288cd12ea7923a58ca090dff492 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Wed, 15 Oct 2025 15:58:08 -0700 Subject: [PATCH 3/9] test(router): update error message assertion after string concat optimization Update test_generate_model_id_with_deployment_model_name to accept the new error message format that results from the list+join optimization. The function still correctly rejects None values with a TypeError, but the error message changed from 'unsupported operand type(s) for +=' to 'expected str instance, NoneType found' due to the implementation change from string concatenation to list joining. --- tests/router_unit_tests/test_router_helper_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py index a31c4d8210f..c2339d9eec5 100644 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ b/tests/router_unit_tests/test_router_helper_utils.py @@ -1411,7 +1411,8 @@ def test_generate_model_id_with_deployment_model_name(model_list): "Expected TypeError when model_group is None - this confirms our fix is needed" ) except TypeError as e: - assert "unsupported operand type(s) for +=" in str(e) + # After optimization, error message changed but still fails appropriately on None + assert "unsupported operand type(s) for +=" in str(e) or "expected str instance, NoneType found" in str(e) print(f"✓ Correctly failed with None model_group (as expected): {e}") except Exception as e: pytest.fail(f"Unexpected error with None model_group: {e}") From 294fb94b1873dd9bd885228915f34aa32ad7b7be Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Wed, 15 Oct 2025 16:25:37 -0700 Subject: [PATCH 4/9] perf(router): use shallow copy instead of deepcopy for model aliases Replace copy.deepcopy() with dict.copy() in _get_all_deployments when creating model aliases. Safe because: 1. Only modifies top-level 'model_name' field (isolated by shallow copy) 2. Nested dicts (litellm_params, model_info) are never modified after return 3. When model_alias=None, returns original dict with NO copy, proving callers expect nested structures to be read-only 4. Key insight: Code that needs to modify nested structures does deepcopy FIRST. This proves the contract is: \"treat returned deployments as read-only for nested fields.\" (see line 6894: copy.deepcopy before modifying litellm_params) Performance: ~10-100x faster than deepcopy on nested dict structures. Tests: All 114 router unit tests pass. --- litellm/router.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5972b06f01a..4616d77b2c9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6257,7 +6257,9 @@ class Router: model_name=model_name, model=model, team_id=team_id ): if model_alias is not None: - alias_model = copy.deepcopy(model) + # Optimized: Use shallow copy since we only modify top-level model_name + # This is much faster than deepcopy for nested dict structures + alias_model = model.copy() alias_model["model_name"] = model_alias returned_models.append(alias_model) else: @@ -6271,7 +6273,8 @@ class Router: model_name=model_name, model=model, team_id=team_id ): if model_alias is not None: - alias_model = copy.deepcopy(model) + # Optimized: Use shallow copy since we only modify top-level model_name + alias_model = model.copy() alias_model["model_name"] = model_alias returned_models.append(alias_model) else: From 6e4da1da65578de5697a11067d90cbebde5cd879 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Wed, 15 Oct 2025 16:41:40 -0700 Subject: [PATCH 5/9] perf(router): optimize model lookups with O(1) data structures Replace O(n) scans with index map lookups and convert model_names to set. Standardize timing calls. --- litellm/router.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 5972b06f01a..05fff0afaf5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -189,7 +189,7 @@ class RoutingArgs(enum.Enum): class Router: - model_names: List = [] + model_names: set = set() cache_responses: Optional[bool] = False default_cache_time_seconds: int = 1 * 60 * 60 # 1 hour tenacity = None @@ -5154,7 +5154,7 @@ class Router: verbose_router_logger.debug( f"\nInitialized Model List {self.get_model_names()}" ) - self.model_names = [m["model_name"] for m in model_list] + self.model_names = {m["model_name"] for m in model_list} # Build model_name index for O(1) lookups self._build_model_name_index(self.model_list) @@ -5360,7 +5360,7 @@ class Router: self._add_model_to_list_and_index_map( model=_deployment, model_id=deployment.model_info.id ) - self.model_names.append(deployment.model_name) + self.model_names.add(deployment.model_name) return deployment def _update_deployment_indices_after_removal( From 56838e2388d92073f95ddcec1287df2ae7d76819 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Wed, 15 Oct 2025 17:52:44 -0700 Subject: [PATCH 6/9] test: ensure model_names is a O(1) datastructure --- tests/router_unit_tests/test_router_index_management.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 04ea9214991..63f9d118349 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -177,3 +177,11 @@ class TestRouterIndexManagement: # Verify: New entry is added assert "claude-3" in router.model_name_to_deployment_indices assert router.model_name_to_deployment_indices["claude-3"] == [0] + + def test_model_names_is_set(self): + """Verify that model_names uses a set for O(1) lookups, not a list (O(n))""" + router = Router(model_list=[]) + + assert isinstance(router.model_names, set), ( + f"model_names should be a set for O(1) lookups, but got {type(router.model_names)}" + ) From 5e929dad2d62e84c1767090fa1f63ebca4784076 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Thu, 16 Oct 2025 09:04:57 -0700 Subject: [PATCH 7/9] test: add static analysis to prevent O(n) linear scans in router Add AST-based test to detect 'for ... in self.model_list' anti-pattern. Enforces use of index maps (model_id_to_deployment_index_map and model_name_to_deployment_indices) for O(1) lookups instead of O(n) iteration. --- .../test_router_index_management.py | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 04ea9214991..28a48604a01 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -1,6 +1,7 @@ import sys import os import pytest +import ast sys.path.insert( 0, os.path.abspath("../..") @@ -177,3 +178,90 @@ class TestRouterIndexManagement: # Verify: New entry is added assert "claude-3" in router.model_name_to_deployment_indices assert router.model_name_to_deployment_indices["claude-3"] == [0] + + def test_no_linear_scans_in_router(self): + """ + Static analysis test to ensure Router doesn't use O(n) linear scans. + + Scans router.py for 'in self.model_list' pattern which indicates + inefficient O(n) iteration instead of using index-based O(1) lookups. + + Methods should use: + - model_id_to_deployment_index_map for O(1) model_id lookups + - model_name_to_deployment_indices for O(1) + O(k) model_name lookups + """ + # Methods that are allowed to iterate through self.model_list + ALLOWED_METHODS = [ + "_get_deployment_by_litellm_model", # Edge case: lookup by litellm_params.model (not indexed) + ] + + # Get path to router.py + router_file = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(__file__))), + "litellm", + "router.py" + ) + + # Read the file + with open(router_file, 'r') as f: + content = f.read() + + # Parse with AST + tree = ast.parse(content) + + # Find violations + violations = [] + ignore_methods = set(ALLOWED_METHODS) + + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef): + method_name = node.name + + # Skip ignored methods + if method_name in ignore_methods: + continue + + # Get source for this method + try: + method_source = ast.get_source_segment(content, node) + if not method_source: + continue + + # Check for the anti-pattern: "in self.model_list" + # This catches: for x in self.model_list, if x in self.model_list, etc. + if "in self.model_list" in method_source: + # Extract the specific line for better error reporting + lines = method_source.split('\n') + pattern_line = None + for line in lines: + if "in self.model_list" in line: + pattern_line = line.strip() + break + + violations.append({ + "method": method_name, + "line": node.lineno, + "pattern": pattern_line or "in self.model_list" + }) + except Exception: + # Skip if we can't get source segment + pass + + # Assert no violations + if violations: + error_msg = "\n".join([ + f" - {v['method']}() at line {v['line']}: {v['pattern']}" + for v in violations + ]) + + pytest.fail( + f"\n{'='*70}\n" + f"Found O(n) linear scan pattern in router.py:\n\n" + f"{error_msg}\n\n" + f"These methods should use index maps instead:\n" + f" - model_id_to_deployment_index_map (for model_id lookups)\n" + f" - model_name_to_deployment_indices (for model_name lookups)\n\n" + f"If a method legitimately needs O(n) iteration, add it to\n" + f"ALLOWED_METHODS in this test method.\n" + f"{'='*70}\n" + ) From 18c5ec8ff735813b702cfbdfeebeaaa66ce2df97 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Thu, 16 Oct 2025 11:12:13 -0700 Subject: [PATCH 8/9] Add missing env key --- docs/my-website/docs/proxy/config_settings.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 4e440857261..627b1bb2374 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -752,6 +752,7 @@ router_settings: | SPEND_LOGS_URL | URL for retrieving spend logs | SPEND_LOG_CLEANUP_BATCH_SIZE | Number of logs deleted per batch during cleanup. Default is 1000 | SSL_CERTIFICATE | Path to the SSL certificate file +| SSL_ECDH_CURVE | ECDH curve for SSL/TLS key exchange (e.g., 'X25519' to disable PQC). | SSL_SECURITY_LEVEL | [BETA] Security level for SSL/TLS connections. E.g. `DEFAULT@SECLEVEL=1` | SSL_VERIFY | Flag to enable or disable SSL certificate verification | SSL_CERT_FILE | Path to the SSL certificate file for custom CA bundle From 8a325530fc7d49154d7e4961a02d4559f5403847 Mon Sep 17 00:00:00 2001 From: AlexsanderHamir Date: Thu, 16 Oct 2025 11:25:03 -0700 Subject: [PATCH 9/9] fix(proxy): change check_file_size_under_limit to accept Collection[str] Fixes mypy type error where router_model_names (a set) was incompatible with List[str] parameter type. Changed check_file_size_under_limit to accept Collection[str] instead of List[str] since the parameter is only used for membership checks, making it more flexible and avoiding unnecessary type conversions. Resolves: proxy/proxy_server.py:5107 arg-type error --- litellm/proxy/common_utils/http_parsing_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 33d432e8695..807b895bd03 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -1,6 +1,6 @@ import json import re -from typing import Any, Dict, List, Optional +from typing import Any, Collection, Dict, List, Optional import orjson from fastapi import Request, UploadFile, status @@ -149,7 +149,7 @@ def _safe_get_request_headers(request: Optional[Request]) -> dict: def check_file_size_under_limit( request_data: dict, file: UploadFile, - router_model_names: List[str], + router_model_names: Collection[str], ) -> bool: """ Check if any files passed in request are under max_file_size_mb