From 5c775dc63ca559af261df0289b022359d69dac83 Mon Sep 17 00:00:00 2001 From: Alberto Perdomo Date: Wed, 28 Oct 2026 16:31:03 +0000 Subject: [PATCH] fix: Reduce count route gate complexity and fix the vLLM auth docstring Signed-off-by: Alberto Perdomo --- .../hosted_vllm/count_tokens/token_counter.py | 2 +- litellm/proxy/proxy_server.py | 15 +++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/litellm/llms/hosted_vllm/count_tokens/token_counter.py b/litellm/llms/hosted_vllm/count_tokens/token_counter.py index 610ba85f120..2fd22648cd3 100644 --- a/litellm/llms/hosted_vllm/count_tokens/token_counter.py +++ b/litellm/llms/hosted_vllm/count_tokens/token_counter.py @@ -52,7 +52,7 @@ def _input_tokens(payload: Mapping[str, JsonValue]) -> int: def _auth_header(api_key: object) -> Mapping[str, str]: - """vLLM commonly runs keyless, so an unset key sends no auth header at all.""" + """Send no auth header when the key is unset, since vLLM commonly runs keyless.""" return {"Authorization": f"Bearer {api_key}"} if isinstance(api_key, str) and api_key else {} diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index af96e0dd689..ba166fdea87 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -14711,13 +14711,18 @@ async def _raise_if_count_route_model_unserved( model: str, requested: str, team_id: str | None, + require_deployment: bool, ) -> None: """Raise 404 not_found_error when routing cannot pick a deployment for ``model``. A model a sibling replica loaded but this replica has not is read through before the rejection, the way route_request recovers from the same miss. """ - if _count_route_model_is_served(llm_router, model, team_id): + if ( + not require_deployment + or llm_router.router_general_settings.pass_through_all_models + or _count_route_model_is_served(llm_router, model, team_id) + ): return from litellm.proxy.common_utils.registry_read_through import model_registry_read_through @@ -15369,12 +15374,10 @@ async def count_request_tokens( if llm_router is not None: # An unserved model on the Anthropic count route is a 404, not a local estimate from an # unrelated tokenizer. pass_through_all_models opts out, as on the completion routes. - pass_through_all_models: Final = llm_router.router_general_settings.pass_through_all_models request_kwargs: Final = dict(_count_route_request_kwargs(user_api_key_dict)) - if require_deployment and not pass_through_all_models: - await _raise_if_count_route_model_unserved( - llm_router, resolution_model, request.model, _count_route_team_id(user_api_key_dict) - ) + await _raise_if_count_route_model_unserved( + llm_router, resolution_model, request.model, _count_route_team_id(user_api_key_dict), require_deployment + ) # get 1 deployment corresponding to the model try: deployment = await llm_router.async_get_available_deployment(