From b4b27203fca2439aa251ca4250ce25a78641793c Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Thu, 5 Feb 2026 10:34:53 -0800 Subject: [PATCH 1/3] perf: cache request.url.path in _get_metadata_variable_name (add_litellm_data_to_request) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cache request.url.path once instead of accessing it 6 times (2 in assistants check + 4 in LITELLM_METADATA_ROUTES loop). Reduces per-call time from 47µs to 20µs (-58%). Also inline the assistants API check to avoid function call overhead. Adds 8 tests for _get_metadata_variable_name covering all return paths. --- litellm/proxy/litellm_pre_call_utils.py | 8 ++-- .../proxy/test_litellm_pre_call_utils.py | 42 +++++++++++++++++++ 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 9be78264e85..0ae2f899b8f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -74,12 +74,14 @@ def _get_metadata_variable_name(request: Request) -> str: For all /thread or /assistant endpoints we need to call this "litellm_metadata" - For ALL other endpoints we call this "metadata + For ALL other endpoints we call this "metadata" """ - if RouteChecks._is_assistants_api_request(request): + path = request.url.path + + if "thread" in path or "assistant" in path: return "litellm_metadata" - if any(route in request.url.path for route in LITELLM_METADATA_ROUTES): + if any(route in path for route in LITELLM_METADATA_ROUTES): return "litellm_metadata" return "metadata" diff --git a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py index da6a5aeab09..6801ecc288e 100644 --- a/tests/test_litellm/proxy/test_litellm_pre_call_utils.py +++ b/tests/test_litellm/proxy/test_litellm_pre_call_utils.py @@ -15,6 +15,7 @@ from litellm.proxy.litellm_pre_call_utils import ( LiteLLMProxyRequestSetup, _get_dynamic_logging_metadata, _get_enforced_params, + _get_metadata_variable_name, _update_model_if_key_alias_exists, add_guardrails_from_policy_engine, add_litellm_data_to_request, @@ -47,6 +48,47 @@ def test_check_if_token_is_service_account(): assert check_if_token_is_service_account(other_metadata_token) == False +class TestGetMetadataVariableName: + """Tests for _get_metadata_variable_name()""" + + def _make_request(self, path: str) -> MagicMock: + request = MagicMock(spec=Request) + request.url.path = path + return request + + def test_returns_litellm_metadata_for_thread_routes(self): + request = self._make_request("/v1/threads/thread_123/messages") + assert _get_metadata_variable_name(request) == "litellm_metadata" + + def test_returns_litellm_metadata_for_assistant_routes(self): + request = self._make_request("/v1/assistants/asst_123") + assert _get_metadata_variable_name(request) == "litellm_metadata" + + def test_returns_litellm_metadata_for_batches_route(self): + request = self._make_request("/v1/batches") + assert _get_metadata_variable_name(request) == "litellm_metadata" + + def test_returns_litellm_metadata_for_messages_route(self): + request = self._make_request("/v1/messages") + assert _get_metadata_variable_name(request) == "litellm_metadata" + + def test_returns_litellm_metadata_for_files_route(self): + request = self._make_request("/v1/files") + assert _get_metadata_variable_name(request) == "litellm_metadata" + + def test_returns_metadata_for_chat_completions(self): + request = self._make_request("/chat/completions") + assert _get_metadata_variable_name(request) == "metadata" + + def test_returns_metadata_for_completions(self): + request = self._make_request("/v1/completions") + assert _get_metadata_variable_name(request) == "metadata" + + def test_returns_metadata_for_embeddings(self): + request = self._make_request("/v1/embeddings") + assert _get_metadata_variable_name(request) == "metadata" + + def test_get_enforced_params_for_service_account_settings(): """ Test that service account enforced params are only added to service account keys From 58291e5e65737d41d9097748fa1c809524f7f71e Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Thu, 5 Feb 2026 13:45:50 -0800 Subject: [PATCH 2/3] perf: skip guardrails processing when not configured (16% faster add_litellm_data_to_request) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove duplicate add_guardrails_from_policy_engine() call that was already being made inside move_guardrails_to_metadata() - Add early-out guard in move_guardrails_to_metadata() to skip all guardrails processing when no guardrails/policies are configured - Remove redundant data.update() wrapper around add_litellm_metadata_from_request_headers() which already modifies data in place Benchmark (3 runs x 2000 req x 1000 concurrency): - add_litellm_data_to_request: 8.87s → 7.45s (-16%) - Guardrails overhead: 1.0s (11.2%) → 0.24s (3.2%) (-76%) --- litellm/proxy/litellm_pre_call_utils.py | 42 ++++++++++++++++--------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0ae2f899b8f..3322860e4fc 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -848,12 +848,10 @@ async def add_litellm_data_to_request( # noqa: PLR0915 ) ) - data.update( - LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( - headers=_headers, - data=data, - _metadata_variable_name=_metadata_variable_name, - ) + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=_headers, + data=data, + _metadata_variable_name=_metadata_variable_name, ) # Add headers to metadata for guardrails to access (fixes #17477) @@ -1108,20 +1106,13 @@ async def add_litellm_data_to_request( # noqa: PLR0915 if disabled_callbacks and isinstance(disabled_callbacks, list): data["litellm_disabled_callbacks"] = disabled_callbacks - # Guardrails from key/team metadata + # Guardrails from key/team metadata and policy engine move_guardrails_to_metadata( data=data, _metadata_variable_name=_metadata_variable_name, user_api_key_dict=user_api_key_dict, ) - # Guardrails from policy engine - add_guardrails_from_policy_engine( - data=data, - metadata_variable_name=_metadata_variable_name, - user_api_key_dict=user_api_key_dict, - ) - # Team Model Aliases _update_model_if_team_alias_exists( data=data, @@ -1462,6 +1453,29 @@ def move_guardrails_to_metadata( - Adds guardrails from policies attached to key/team metadata - Adds guardrails from policy engine based on team/key/model context """ + # Early-out: skip all guardrails processing when nothing is configured + key_metadata = user_api_key_dict.metadata + team_metadata = user_api_key_dict.team_metadata + + has_key_config = key_metadata and ( + "guardrails" in key_metadata or "policies" in key_metadata + ) + has_team_config = team_metadata and ( + "guardrails" in team_metadata or "policies" in team_metadata + ) + has_request_config = ( + "guardrails" in data or "guardrail_config" in data or "policies" in data + ) + + # Only check policy engine if no local config (avoid import + registry lookup) + if not (has_key_config or has_team_config or has_request_config): + from litellm.proxy.policy_engine.policy_registry import get_policy_registry + + if not get_policy_registry().is_initialized(): + # Nothing configured anywhere - clean up request body fields and return + data.pop("policies", None) + return + # Check key-level guardrails _add_guardrails_from_key_or_team_metadata( key_metadata=user_api_key_dict.metadata, From 1ba10cc22c111b9372c9ef962ae160cf65887e9f Mon Sep 17 00:00:00 2001 From: Ryan Crabbe Date: Thu, 5 Feb 2026 13:52:40 -0800 Subject: [PATCH 3/3] perf: eliminate duplicate dict(request.headers) calls - Create _raw_headers once at start, reuse for SecretFields - Reuse _headers from clean_headers() instead of creating new dict (clean_headers already removes authorization) - Eliminates 2 of 3 dict(request.headers) calls per request --- litellm/proxy/litellm_pre_call_utils.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 3322860e4fc..d593ac00e92 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -812,7 +812,8 @@ async def add_litellm_data_to_request( # noqa: PLR0915 from litellm.proxy.proxy_server import llm_router, premium_user from litellm.types.proxy.litellm_pre_call_utils import SecretFields - _headers = clean_headers( + _raw_headers: Dict[str, str] = dict(request.headers) + _headers: Dict[str, str] = clean_headers( request.headers, litellm_key_header_name=( general_settings.get("litellm_key_header_name") @@ -878,7 +879,7 @@ async def add_litellm_data_to_request( # noqa: PLR0915 if "user" not in data: data["user"] = user - data["secret_fields"] = SecretFields(raw_headers=dict(request.headers)) + data["secret_fields"] = SecretFields(raw_headers=_raw_headers) ## Dynamic api version (Azure OpenAI endpoints) ## try: @@ -1022,10 +1023,6 @@ async def add_litellm_data_to_request( # noqa: PLR0915 ] = user_api_key_dict.user_max_budget data[_metadata_variable_name]["user_api_key_metadata"] = user_api_key_dict.metadata - _headers = dict(request.headers) - _headers.pop( - "authorization", None - ) # do not store the original `sk-..` api key in the db data[_metadata_variable_name]["headers"] = _headers data[_metadata_variable_name]["endpoint"] = str(request.url) @@ -1077,7 +1074,7 @@ async def add_litellm_data_to_request( # noqa: PLR0915 # Check if using tag based routing tags = LiteLLMProxyRequestSetup.add_request_tag_to_metadata( llm_router=llm_router, - headers=dict(request.headers), + headers=_headers, data=data, )