From f3a9721adcb6e54d5c1e40df2f84787233855f3d Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 23 Mar 2026 22:38:54 +0530 Subject: [PATCH] Fix greptile reviews and mock test --- docs/my-website/docs/proxy/load_balancing.md | 15 +++++++++++++ litellm/proxy/litellm_pre_call_utils.py | 23 +++++++++++++++----- litellm/router.py | 6 +++++ 3 files changed, 39 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/load_balancing.md b/docs/my-website/docs/proxy/load_balancing.md index 93f3d944340..a311d688b55 100644 --- a/docs/my-website/docs/proxy/load_balancing.md +++ b/docs/my-website/docs/proxy/load_balancing.md @@ -377,6 +377,21 @@ For router internals: when a `team_id` is in scope, optimized lookups key off `( If a stale alias is detected and the bypass is **not** enabled, the proxy may emit a **one-time** warning in logs explaining that sibling deployments may be unreachable until the flag is set or aliases are cleaned up. +### Team-scoped models and legacy `model_aliases` {#team-scoped-models-and-legacy-model_aliases} + +Team-scoped deployments are identified by `model_info.team_id` and `model_info.team_public_model_name`. Requests should use the **public** model name; the router resolves all sibling deployments (same public name, different `api_base` / `order`, etc.) for routing, failover, and deployment `order`. + +For router internals: when a `team_id` is in scope, optimized lookups key off `(team_id, team_public_model_name)`. If code passes an internal deployment id (e.g. `model_name__`) instead of the public name, routing still works via the usual deployment-name paths, but the team-specific fast path applies only to the public name. + +**Legacy teams:** Older proxy versions could persist `model_aliases` on the team row mapping a public name to a single internal deployment id (`model_name__`). On each request, pre-call logic may still rewrite `model` to that internal name **before** routing, which collapses to one deployment and can make newer sibling deployments unreachable. + +**Migration options:** + +1. **Recommended for upgrades:** Set environment variable `LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true` so that when sibling team deployments exist for the public name, the stale alias rewrite is skipped and team-scoped routing (including `order` and failover) applies. See the [Environment variables](./config_settings) table in the proxy settings doc. +2. **Data cleanup:** Remove obsolete `model_aliases` entries for team public names from the team record in the database so only `team_public_model_name` + team model list drive access. + +If a stale alias is detected and the bypass is **not** enabled, the proxy may emit a **one-time** warning in logs explaining that sibling deployments may be unreachable until the flag is set or aliases are cleaned up. + ### When You'll See Load Balancing in Action **Immediate Effects:** diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 4a12a0a5774..64ef2405002 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -37,6 +37,7 @@ from litellm.types.utils import ( ) service_logger_obj = ServiceLogging() # used for tracking latency on OTEL +_STALE_TEAM_ALIAS_WARNING_KEYS: set[str] = set() if TYPE_CHECKING: @@ -1321,16 +1322,28 @@ def _update_model_if_team_alias_exists( ) # Check if the alias points to a team-scoped UUID name # (format: "model_name_{team_id}_{uuid}") - if enable_stale_alias_bypass and aliased_target.startswith( + is_stale_team_alias = aliased_target.startswith( f"model_name_{user_api_key_dict.team_id}_" - ): + ) + if is_stale_team_alias and llm_router: # This is a stale alias from pre-PR deployments. # Check if current team deployments exist for the public name. - if llm_router: - key = (user_api_key_dict.team_id, _model) - if key in llm_router.team_model_to_deployment_indices: + key = (user_api_key_dict.team_id, _model) + if key in llm_router.team_model_to_deployment_indices: + if enable_stale_alias_bypass: # Team deployments exist; skip stale alias return + warning_key = f"{user_api_key_dict.team_id}:{_model}:{aliased_target}" + if warning_key not in _STALE_TEAM_ALIAS_WARNING_KEYS: + _STALE_TEAM_ALIAS_WARNING_KEYS.add(warning_key) + verbose_proxy_logger.warning( + "Stale team model alias detected for model='%s', team_id='%s'. " + "New sibling deployments may be unreachable. " + "Set LITELLM_ENABLE_TEAM_STALE_ALIAS_BYPASS=true to enable " + "team-scoped sibling routing.", + _model, + user_api_key_dict.team_id, + ) data["model"] = aliased_target return diff --git a/litellm/router.py b/litellm/router.py index d7f5d42eac7..64d29d8bceb 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -8256,6 +8256,12 @@ class Router: if team_id specified, only return team-specific models Optimized with O(1) index lookup instead of O(n) linear scan. + + Note: when team_id is provided, O(1) lookup in + `team_model_to_deployment_indices` only applies when `model_name` is the + team public model name. If a caller passes an internal deployment model + name (for example, `model_name__`), this method falls back + to the standard model-name index / scan path. """ returned_models: List[DeploymentTypedDict] = []