litellm/litellm/proxy/auth/auth_checks.py

# What is this?
## Common auth checks between jwt + key based auth
"""
Got Valid Token from Cache, DB
Run checks for:

1. If user can call model
2. If user is in budget
3. If end_user ('user' passed to /chat/completions, /embeddings endpoint) is in budget
"""

import asyncio
import math
import re
import time
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, cast

from fastapi import HTTPException, Request, status
from pydantic import BaseModel
from typing_extensions import ReadOnly, TypedDict

import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching.dual_cache import LimitedSizeOrderedDict
from litellm.constants import (
    CLI_JWT_EXPIRATION_HOURS,
    CLI_SESSION_KEY_PREFIX,
    DEFAULT_ACCESS_GROUP_CACHE_TTL,
    DEFAULT_IN_MEMORY_TTL,
    DEFAULT_MAX_RECURSE_DEPTH,
    EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE,
    END_USER_RESTRICTED_REGISTRY_MAX_SIZE,
    REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
    TAG_REGISTRY_MAX_SIZE,
)
from litellm.litellm_core_utils.dd_tracing import tracer
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.proxy._types import (
    RBAC_ROLES,
    CallInfo,
    LiteLLM_AccessGroupTable,
    LiteLLM_BudgetTable,
    LiteLLM_EndUserTable,
    Litellm_EntityType,
    LiteLLM_JWTAuth,
    LiteLLM_ManagedVectorStoresTable,
    LiteLLM_ObjectPermissionTable,
    LiteLLM_OrganizationMembershipTable,
    LiteLLM_OrganizationTable,
    LiteLLM_ProjectTableCachedObj,
    LiteLLM_TagTable,
    LiteLLM_TeamMembership,
    LiteLLM_TeamTable,
    LiteLLM_TeamTableCachedObj,
    LiteLLM_UserTable,
    LiteLLMRoutes,
    LitellmUserRoles,
    NewTeamRequest,
    ProxyErrorTypes,
    ProxyException,
    RoleBasedPermissions,
    SpecialModelNames,
    UserAPIKeyAuth,
)
from litellm.proxy.auth.budget_throttle import (
    budget_throttle_percentage,
    should_throttle_budget_exceeded,
)
from litellm.proxy.auth.route_checks import RouteChecks
from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import publish_auth_cache_invalidation
from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
from litellm.proxy.common_utils.http_parsing_utils import (
    _safe_get_request_headers,
    _safe_get_request_query_params,
)
from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
from litellm.proxy.common_utils.user_api_key_cache import (
    END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL,
    TAG_REGISTRY_OVERFLOW_SENTINEL,
    UserApiKeyCache,
    end_user_cache_key,
    end_user_restricted_registry_cache_key,
    get_management_object_ttl,
    object_permission_cache_key,
    tag_cache_key,
    tag_registry_cache_key,
    team_membership_auth_cache_key,
    team_membership_reservation_cache_key,
)
from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.guardrails.tool_name_extraction import (
    TOOL_CAPABLE_CALL_TYPES,
    extract_request_tool_names,
)
from litellm.proxy.route_llm_request import route_request
from litellm.proxy.spend_tracking.budget_reservation import get_budget_window_start
from litellm.proxy.utils import PrismaClient, ProxyLogging, log_db_metrics
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.prisma_protocols import RowT_co
from litellm.repositories.project_repository import ProjectRepository
from litellm.repositories.table_repositories import (
    AccessGroupRepository,
    EndUserRepository,
    JWTKeyMappingRepository,
    ManagedVectorStoresRepository,
    TagRepository,
    TeamMembershipRepository,
)
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.router import Router
from litellm.utils import get_utc_datetime

from .auth_checks_organization import (
    add_team_org_context_to_request_body,
    organization_role_based_access_check,
)
from .auth_utils import get_model_from_request, get_request_route_template

if TYPE_CHECKING:
    from opentelemetry.trace import Span as _Span

    Span = _Span
else:
    Span = Any


class _PrismaDictableRow(Protocol):
    def dict(self) -> Mapping[str, object]: ...


class _PrismaJWTKeyMappingRow(Protocol):
    token: str


class _PrismaModelDumpRow(Protocol):
    def model_dump(self) -> Mapping[str, object]: ...


class _PrismaTeamRow(Protocol):
    def dict(self) -> Mapping[str, object]: ...

    def model_dump(self) -> Mapping[str, object]: ...


class _PrismaVectorStoreRow(Protocol):
    def dict(self) -> Mapping[str, object]: ...

    def model_dump(self) -> Mapping[str, object]: ...

    def __iter__(self) -> Iterator[tuple[str, object]]: ...


class _PrismaUserRow(Protocol):
    user_id: str
    organization_memberships: Sequence[LiteLLM_OrganizationMembershipTable | None] | None

    def __iter__(self) -> Iterator[tuple[str, object]]: ...


class _PrismaAuthTable(Protocol[RowT_co]):
    async def find_unique(
        self, *, where: Mapping[str, object], include: Mapping[str, object] | None = None
    ) -> RowT_co | None: ...

    async def find_first(
        self, *, where: Mapping[str, object], include: Mapping[str, object] | None = None
    ) -> RowT_co | None: ...

    async def find_many(
        self,
        *,
        where: Mapping[str, object] | None = None,
        include: Mapping[str, object] | None = None,
        take: int | None = None,
    ) -> Sequence[RowT_co]: ...

    async def update(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> RowT_co | None: ...

    async def create(self, *, data: Mapping[str, object], include: Mapping[str, object] | None = None) -> RowT_co: ...


class _PrismaTableHolder(Protocol[RowT_co]):
    @property
    def table(self) -> _PrismaAuthTable[RowT_co]: ...


def _dictable_table(repo: _PrismaTableHolder[_PrismaDictableRow]) -> _PrismaAuthTable[_PrismaDictableRow]:
    return repo.table


def _jwt_key_mapping_table(
    repo: _PrismaTableHolder[_PrismaJWTKeyMappingRow],
) -> _PrismaAuthTable[_PrismaJWTKeyMappingRow]:
    return repo.table


def _model_dump_table(repo: _PrismaTableHolder[_PrismaModelDumpRow]) -> _PrismaAuthTable[_PrismaModelDumpRow]:
    return repo.table


def _team_table(repo: _PrismaTableHolder[_PrismaTeamRow]) -> _PrismaAuthTable[_PrismaTeamRow]:
    return repo.table


def _vector_store_table(repo: _PrismaTableHolder[_PrismaVectorStoreRow]) -> _PrismaAuthTable[_PrismaVectorStoreRow]:
    return repo.table


def _user_table(repo: _PrismaTableHolder[_PrismaUserRow]) -> _PrismaAuthTable[_PrismaUserRow]:
    return repo.table


def _object_permission_table(
    repo: _PrismaTableHolder[LiteLLM_ObjectPermissionTable],
) -> _PrismaAuthTable[LiteLLM_ObjectPermissionTable]:
    return repo.table


class _PrismaTagRow(Protocol):
    tag_name: str

    def dict(self) -> Mapping[str, object]: ...


def _tag_table(repo: _PrismaTableHolder[_PrismaTagRow]) -> _PrismaAuthTable[_PrismaTagRow]:
    return repo.table


class _PrismaEndUserRow(Protocol):
    user_id: str

    def dict(self) -> Mapping[str, object]: ...


def _end_user_table(repo: _PrismaTableHolder[_PrismaEndUserRow]) -> _PrismaAuthTable[_PrismaEndUserRow]:
    return repo.table


class _RawCacheRead(Protocol):
    async def async_get_cache(self, *, key: str) -> object: ...


def _raw_cache(cache: _RawCacheRead) -> _RawCacheRead:
    return cache


def _typed_request_body(request_body: dict) -> Mapping[str, object]:
    return request_body


class _JsonLoadsObj(Protocol):
    def __call__(self, data: str) -> object: ...


def _typed_json_loads(fn: _JsonLoadsObj) -> _JsonLoadsObj:
    return fn


_safe_json_loads_obj: Final = _typed_json_loads(safe_json_loads)


last_db_access_time: Final = LimitedSizeOrderedDict(max_size=100)
db_cache_expiry: Final = DEFAULT_IN_MEMORY_TTL  # refresh every 5s

all_routes: Final = LiteLLMRoutes.openai_routes.value + LiteLLMRoutes.management_routes.value


def _log_budget_lookup_failure(entity: str, error: Exception) -> None:
    """
    Log a warning when budget lookup fails; cache will not be populated.

    Skips logging for expected "user not found" cases (bare Exception from
    get_user_object when user_id_upsert=False). Adds a schema migration hint
    when the error appears schema-related.
    """
    # Skip logging for expected "user not found" - not caching is correct
    if str(error) == "" and type(error).__name__ == "Exception":
        return
    err_str: Final = str(error).lower()
    hint = ""
    if any(x in err_str for x in ("column", "schema", "does not exist", "prisma", "migrate")):
        hint = " Run `prisma db push` or `prisma migrate deploy` to fix schema mismatches."
    verbose_proxy_logger.error(
        "Budget lookup failed for %s; cache will not be populated. Each request will hit the database. Error: %s.%s",
        entity,
        error,
        hint,
    )


def _get_router_zero_cost_cache(llm_router: Router) -> dict[str, bool] | None:
    """
    Return the router's per-instance zero-cost cache, or ``None`` for objects
    that don't expose one (e.g. ``MagicMock`` stand-ins in unit tests).

    The cache lives on the ``Router`` instance so it:
        * is invalidated by ``Router._invalidate_model_group_info_cache`` on
          any model add/remove/upsert (including in-place pricing changes via
          ``/model/update``, which go through ``upsert_deployment``);
        * dies with the router itself — no risk of CPython reusing the
          previous router's ``id()`` and serving its cached entries.
    """
    cache: Final = getattr(llm_router, "_zero_cost_cache", None)
    return cache if isinstance(cache, dict) else None


def _is_model_cost_zero(model: str | list[str] | None, llm_router: Router | None) -> bool:
    """
    Check if a model has zero cost (no configured pricing).

    Uses the router's get_model_group_info method to get pricing information.

    Args:
        model: The model name or list of model names
        llm_router: The LiteLLM router instance

    Returns:
        bool: True if all costs for the model are zero, False otherwise
    """
    if model is None or llm_router is None:
        return False

    # Handle list of models
    model_list: Final = [model] if isinstance(model, str) else model

    zero_cost_cache: Final = _get_router_zero_cost_cache(llm_router)

    for model_name in model_list:
        if zero_cost_cache is not None:
            cached = zero_cost_cache.get(model_name)
            if cached is not None:
                if cached is False:
                    return False
                continue
        try:
            # Use router's get_model_group_info method directly for better reliability
            model_group_info = llm_router.get_model_group_info(model_group=model_name)

            if model_group_info is None:
                # Model not found or no pricing info available
                # Conservative approach: assume it has cost
                verbose_proxy_logger.debug("No model group info found for %s, assuming it has cost", model_name)
                if zero_cost_cache is not None:
                    zero_cost_cache[model_name] = False
                return False

            # Check costs for this model
            # Only allow bypass if BOTH costs are explicitly set to 0 (not None)
            input_cost = model_group_info.input_cost_per_token
            output_cost = model_group_info.output_cost_per_token

            # If costs are not explicitly configured (None), assume it has cost
            if input_cost is None or output_cost is None:
                verbose_proxy_logger.debug(
                    "Model %s has undefined cost (input: %s, output: %s), assuming it has cost",
                    model_name,
                    input_cost,
                    output_cost,
                )
                if zero_cost_cache is not None:
                    zero_cost_cache[model_name] = False
                return False

            # If either cost is non-zero, return False
            if input_cost > 0 or output_cost > 0:
                verbose_proxy_logger.debug(
                    "Model %s has non-zero cost (input: %s, output: %s)", model_name, input_cost, output_cost
                )
                if zero_cost_cache is not None:
                    zero_cost_cache[model_name] = False
                return False

            # Costs are 0 — verify this is from explicit configuration,
            # not from defaulted sparse auto-registration entries.
            # See: https://github.com/BerriAI/litellm/issues/24770
            safe_name = str(model_name).replace("\n", "").replace("\r", "")
            if not _is_cost_explicitly_configured(model_name, llm_router):
                verbose_proxy_logger.debug(
                    "Model %s has zero cost but no explicit cost "
                    "configuration in model_cost entry — treating as unknown "
                    "cost (enforce budget)",
                    safe_name,
                )
                if zero_cost_cache is not None:
                    zero_cost_cache[model_name] = False
                return False

            if _has_ptu_flat_cost(model_name, llm_router):
                verbose_proxy_logger.debug(
                    "Model %s prices reserved PTU capacity as a flat cost, so its zero per-token "
                    "rate is not a free model (enforce budget)",
                    safe_name,
                )
                if zero_cost_cache is not None:
                    zero_cost_cache[model_name] = False
                return False

            verbose_proxy_logger.debug(
                "Model %s has zero cost explicitly configured (input: %s, output: %s)",
                safe_name,
                input_cost,
                output_cost,
            )
            if zero_cost_cache is not None:
                zero_cost_cache[model_name] = True

        except Exception as e:
            # If we can't determine the cost, assume it has cost (conservative approach)
            verbose_proxy_logger.debug("Error checking cost for model %s: %s, assuming it has cost", model_name, e)
            return False

    # All models checked have zero cost
    return True


_NO_MODEL_INFO: Final[Mapping[str, object]] = MappingProxyType({})


def _has_ptu_flat_cost(model: str, llm_router: "Router") -> bool:
    """Whether any deployment in the model group bills reserved PTU capacity as a flat cost.

    Such a deployment carries an explicit zero per-token price so the flat cost is not charged
    twice, which otherwise reads here as a free model and waives every budget check for it.
    """
    for deployment in llm_router.model_list:
        if deployment.get("model_name") != model:
            continue
        model_info = deployment.get("model_info") or _NO_MODEL_INFO
        if model_info.get("ptu_count") is not None and model_info.get("cost_per_ptu_per_hour") is not None:
            return True
    return False


def _is_cost_explicitly_configured(model: str, llm_router: "Router") -> bool:
    """
    Check if any deployment in the model group has cost fields explicitly
    set in its litellm.model_cost entry.

    When Router._create_deployment() registers a model not in the global
    cost map, it creates a sparse entry like {"id": "<hash>"} with no cost
    fields. _get_model_info_helper() then defaults missing costs to 0.
    This function detects that scenario by checking the raw model_cost entry.
    """
    for deployment in llm_router.model_list:
        if deployment.get("model_name") != model:
            continue
        model_id = deployment.get("model_info", {}).get("id")
        if model_id is None:
            continue
        raw_entry = litellm.model_cost.get(model_id, {})
        if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
            return True
    return False


_EMPTY_COST_ENTRY: Final[Mapping[str, object]] = MappingProxyType({})


def _is_positive_cost(value: object) -> bool:
    return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0


def _entry_has_priced_metric(entry: Mapping[str, object]) -> bool:
    if entry.get("tiered_pricing") is not None:
        return True
    for key, value in entry.items():
        if "cost_per" not in key:
            continue
        if _is_positive_cost(value):
            return True
        if isinstance(value, dict) and any(_is_positive_cost(nested) for nested in value.values()):
            return True
    return False


def _entry_declares_price(entry: Mapping[str, object]) -> bool:
    return any("cost_per" in key or key == "tiered_pricing" for key in entry)


def _model_group_has_pricing(model: str, llm_router: "Router") -> bool:
    """
    A model group counts as priced when a deployment overrides any *cost_per* field or
    tiered_pricing in its litellm_params, even at zero, or when its resolved model info carries
    tiered_pricing or a positive price on any billed metric (tokens, characters, seconds, pages,
    images, queries, ...), so models billed by a non-token metric are not treated as unpriced.
    """
    for deployment in llm_router.get_model_list(model_name=model) or ():
        litellm_params = deployment.get("litellm_params") or _EMPTY_COST_ENTRY
        if _entry_declares_price(litellm_params):
            return True

        model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
        if model_id is None:
            continue

        model_info = llm_router.get_deployment_model_info(
            model_id=model_id, model_name=litellm_params.get("model") or ""
        )
        if model_info is not None and _entry_has_priced_metric(model_info):
            return True

    return False


def _group_declares_explicit_cost(model: str, llm_router: "Router") -> bool:
    """
    Alias-aware counterpart to ``_is_cost_explicitly_configured``, which resolves the model group
    the same way ``_model_group_has_pricing`` does. A deployment that prices itself through its
    ``model_info`` block lands in the cost map under its deployment id rather than in its
    litellm_params, and reaching that entry through the router's own resolution keeps an alias
    pointing at such a group from being read as unpriced.
    """
    for deployment in llm_router.get_model_list(model_name=model) or ():
        model_id = (deployment.get("model_info") or _EMPTY_COST_ENTRY).get("id")
        if model_id is None:
            continue
        raw_entry = litellm.model_cost.get(model_id, _EMPTY_COST_ENTRY)
        if "input_cost_per_token" in raw_entry or "output_cost_per_token" in raw_entry:
            return True
    return False


def model_has_no_cost_mapping(model: str | None, llm_router: Router | None) -> bool:
    if not model or llm_router is None:
        return False

    if llm_router.get_model_group_info(model_group=model) is None:
        return False

    if _model_group_has_pricing(model=model, llm_router=llm_router):
        return False

    return not _group_declares_explicit_cost(model=model, llm_router=llm_router)


def _unpriced_models_in_request(model: str | list[str] | None, llm_router: Router | None) -> tuple[str, ...]:
    candidates: Final = (model,) if isinstance(model, str) else tuple(model or ())
    return tuple(
        candidate for candidate in candidates if model_has_no_cost_mapping(model=candidate, llm_router=llm_router)
    )


def _unpriced_models_block_message(models: tuple[str, ...]) -> str:
    names: Final = ", ".join(f"'{model}'" for model in models)
    subject: Final = f"Model {names} has" if len(models) == 1 else f"Models {names} have"
    return (
        f"{subject} no pricing in the cost map, so litellm cannot price the request. "
        "Requests for unpriced models are blocked because 'block_requests_for_models_without_pricing' "
        "is enabled. Add pricing (input_cost_per_token/output_cost_per_token) to allow the request."
    )


async def _run_project_checks(
    project_object: LiteLLM_ProjectTableCachedObj | None,
    _model: str | list[str] | None,
    llm_router: Router | None,
    skip_budget_checks: bool,
    valid_token: UserAPIKeyAuth | None,
    proxy_logging_obj: ProxyLogging,
) -> None:
    """
    Run all project-level checks: blocked, model access, budget, soft budget.
    Extracted from common_checks() to keep statement count manageable.
    """
    if project_object is None:
        return

    # 1.1. If project is blocked
    if project_object.blocked is True:
        raise Exception(
            f"Project={project_object.project_id} is blocked. Update via `/project/update` if you're an admin."
        )

    # 2.2 If project can call model
    if _model and len(project_object.models) > 0:
        can_project_access_model(
            model=_model,
            project_object=project_object,
            llm_router=llm_router,
        )

    if not skip_budget_checks:
        # 3.0.2. If project is in budget
        await _project_max_budget_check(
            project_object=project_object,
            valid_token=valid_token,
            proxy_logging_obj=proxy_logging_obj,
        )

        # 3.0.3. If project is over soft budget (alert only, doesn't block)
        await _project_soft_budget_check(
            project_object=project_object,
            valid_token=valid_token,
            proxy_logging_obj=proxy_logging_obj,
        )


def _enforce_user_param_check(general_settings: dict, request: Request, request_body: dict, route: str) -> None:
    if not general_settings.get("enforce_user_param", False):
        return

    http_method: Final = request.method if hasattr(request, "method") else None
    is_post_method: Final = http_method and http_method.upper() == "POST"
    is_openai_route: Final = RouteChecks.is_llm_api_route(route=route)
    is_mcp_route: Final = route in LiteLLMRoutes.mcp_routes.value or RouteChecks.check_route_access(
        route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value
    )

    if is_post_method and is_openai_route and not is_mcp_route and "user" not in request_body:
        raise Exception(f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}")


def _reject_clientside_metadata_tags_check(general_settings: dict, request_body: dict, route: str) -> None:
    if not general_settings.get("reject_clientside_metadata_tags", False):
        return

    if (
        RouteChecks.is_llm_api_route(route=route)
        and "metadata" in request_body
        and isinstance(request_body["metadata"], dict)
        and "tags" in request_body["metadata"]
    ):
        raise ProxyException(
            message=f"Client-side 'metadata.tags' not allowed in request. 'reject_clientside_metadata_tags'={general_settings['reject_clientside_metadata_tags']}. Tags can only be set via API key metadata.",
            type=ProxyErrorTypes.bad_request_error,
            param="metadata.tags",
            code=status.HTTP_400_BAD_REQUEST,
        )


def _global_proxy_budget_check(global_proxy_spend: float | None, skip_budget_checks: bool, route: str) -> None:
    if (
        litellm.max_budget > 0
        and not skip_budget_checks
        and global_proxy_spend is not None
        and RouteChecks.is_llm_api_route(route=route)
        and route != "/v1/models"
        and route != "/models"
    ):
        if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget:
            raise litellm.BudgetExceededError(
                current_cost=global_proxy_spend,
                max_budget=litellm.max_budget,
                entity_type=Litellm_EntityType.PROXY.value,
            )


_GUARDRAIL_MODIFICATION_KEYS: Final[tuple] = (
    "guardrails",
    "disable_global_guardrails",
    "disable_global_guardrail",
    "opted_out_global_guardrails",
)


def _guardrail_modification_check(request_body: Mapping[str, object], team_object: LiteLLM_TeamTable | None) -> None:
    """
    Reject user-supplied metadata flags that would modify guardrail behavior
    unless the team has explicit permission. Checked keys include the plural
    ``guardrails`` list plus the per-request toggles that influence whether
    default-on guardrails run (``disable_global_guardrails``,
    ``disable_global_guardrail`` singular, and ``opted_out_global_guardrails``).

    User-supplied values for the bypass toggles are also silently ignored by
    ``_get_admin_metadata`` at read time; this check adds defense in depth by
    failing loudly at the auth layer so operators see an explicit 403 instead
    of a confusing silent-ignore.
    """
    from litellm.proxy.guardrails.guardrail_helpers import can_modify_guardrails

    def _coerce_to_dict(container: object) -> dict | None:
        """Accept dict or JSON-string (from multipart/form-data or extra_body).

        Without this, an attacker can smuggle guardrail keys past the check by
        sending ``{"metadata": "{\\"disable_global_guardrails\\": true}"}`` —
        ``isinstance(dict)`` on the string returns False, the check returns
        no-modification, and ``add_litellm_data_to_request`` parses the string
        to a dict downstream.
        """
        if isinstance(container, dict):
            return container
        if isinstance(container, str):
            parsed: Final = _safe_json_loads_obj(container)
            return parsed if isinstance(parsed, dict) else None
        return None

    def _user_requested_modification(container: object) -> bool:
        coerced: Final = _coerce_to_dict(container)
        if coerced is None:
            return False
        return any(key in coerced for key in _GUARDRAIL_MODIFICATION_KEYS)

    # Check both metadata keys — callers can populate either depending on the
    # endpoint. Cover the top-level too so root-level injection is rejected.
    modifies: Final = (
        _user_requested_modification(request_body.get("metadata"))
        or _user_requested_modification(request_body.get("litellm_metadata"))
        or _user_requested_modification(request_body)
    )
    if not modifies:
        return

    if not can_modify_guardrails(team_object):
        raise HTTPException(
            status_code=403,
            detail={"error": "Your team does not have permission to modify guardrails."},
        )


async def check_tools_allowlist(
    request_body: dict,
    valid_token: UserAPIKeyAuth | None,
    team_object: LiteLLM_TeamTable | None,
    route: str,
) -> None:
    """
    Enforce key/team tool allowlist (metadata.allowed_tools). No DB in hot path —
    effective allowlist is read from valid_token.metadata and valid_token.team_metadata.
    Raises ProxyException with tool_access_denied if a tool is not allowed.
    """
    from litellm.litellm_core_utils.api_route_to_call_types import (
        get_call_types_for_route,
    )

    if valid_token is None:
        return
    call_types: Final = get_call_types_for_route(route)
    if not call_types or not any(ct.value in TOOL_CAPABLE_CALL_TYPES for ct in call_types):
        return
    tool_names: Final = extract_request_tool_names(route, request_body)
    if not tool_names:
        return
    key_meta: Final = (valid_token.metadata or {}) if isinstance(valid_token.metadata, dict) else {}
    team_meta: Final = (valid_token.team_metadata or {}) if isinstance(valid_token.team_metadata, dict) else {}
    key_allowed: Final = key_meta.get("allowed_tools")
    team_allowed: Final = team_meta.get("allowed_tools")
    effective: Final = key_allowed if (isinstance(key_allowed, list) and len(key_allowed) > 0) else team_allowed
    if not isinstance(effective, list) or len(effective) == 0:
        return
    allowed_set: Final = {str(t) for t in effective}
    disallowed: Final = [n for n in tool_names if n not in allowed_set]
    if disallowed:
        raise ProxyException(
            message=f"Tool(s) {disallowed} are not in the allowed tools list for this key/team.",
            type=ProxyErrorTypes.tool_access_denied,
            param="tools",
            code=status.HTTP_403_FORBIDDEN,
        )


# Read-only discovery routes that incur no spend. Kept narrower than info_routes so an exhausted
# budget cannot reach side-effectful routes like /health/services (Slack/email/webhook). See #27923.
MODEL_DISCOVERY_ROUTES: Final = frozenset(
    {
        "/v1/models",
        "/models",
        "/model/info",
        "/v1/model/info",
        "/v2/model/info",
        "/model_group/info",
    }
)

BUDGET_ENFORCED_SIDE_EFFECT_ROUTES: Final = frozenset(
    {
        "/health",
        "/health/services",
        "/health/test_connection",
    }
)


async def common_checks(
    request_body: dict,
    team_object: LiteLLM_TeamTable | None,
    user_object: LiteLLM_UserTable | None,
    end_user_object: LiteLLM_EndUserTable | None,
    global_proxy_spend: float | None,
    general_settings: dict,
    route: str,
    llm_router: Router | None,
    proxy_logging_obj: ProxyLogging,
    valid_token: UserAPIKeyAuth | None,
    request: Request,
    skip_budget_checks: bool = False,
    project_object: LiteLLM_ProjectTableCachedObj | None = None,
) -> bool:
    """
    Common checks across jwt + key-based auth.

    1. If team is blocked
    1.1. If project is blocked
    2. If team can call model
    2.2 If project can call model
    3. If team is in budget
    3.0.2. If project is in budget
    3.0.3. If project is over soft budget (alert only)
    4. If user passed in (JWT or key.user_id) - is in budget
    5. If end_user (either via JWT or 'user' passed to /chat/completions, /embeddings endpoint) is in budget
    6. [OPTIONAL] If 'enforce_end_user' enabled - did developer pass in 'user' param for openai endpoints
    7. [OPTIONAL] If 'litellm.max_budget' is set (>0), is proxy under budget
    8. [OPTIONAL] If guardrails modified - is request allowed to change this
    9. Check if request body is safe
    10. [OPTIONAL] Organization checks - is user_object.organization_id is set, run these checks
    11. [OPTIONAL] Vector store checks - is the object allowed to access the vector store
    """
    from litellm.proxy.proxy_server import prisma_client, user_api_key_cache

    _model: Final[str | list[str] | None] = get_model_from_request(
        request_data=request_body,
        route=route,
        request_headers=_safe_get_request_headers(request=request),
        request_query_params=_safe_get_request_query_params(request=request),
        llm_router=llm_router,
        request=request,
    )

    skip_all_budget_checks: Final = skip_budget_checks or (
        route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES
        and (route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route))
    )

    unpriced_models: Final = (
        _unpriced_models_in_request(model=_model, llm_router=llm_router)
        if litellm.block_requests_for_models_without_pricing and RouteChecks.is_llm_api_route(route=route)
        else ()
    )
    if unpriced_models:
        raise ProxyException(
            message=_unpriced_models_block_message(unpriced_models),
            type=ProxyErrorTypes.model_cost_map_missing,
            param="model",
            code=status.HTTP_403_FORBIDDEN,
        )

    # 1. If team is blocked
    if team_object is not None and team_object.blocked is True:
        raise Exception(f"Team={team_object.team_id} is blocked. Update via `/team/unblock` if you're an admin.")

    # 2. If team can call model (or key's access_group_ids grant it)
    if _model and team_object:
        with tracer.trace("litellm.proxy.auth.common_checks.can_team_access_model"):
            try:
                await can_team_access_model(
                    model=_model,
                    team_object=team_object,
                    llm_router=llm_router,
                    team_model_aliases=(valid_token.team_model_aliases if valid_token else None),
                )
            except ProxyException as team_denial:
                if team_denial.type != ProxyErrorTypes.team_model_access_denied:
                    raise
                if not await _key_access_group_grants_model(
                    model=_model,
                    valid_token=valid_token,
                    team_object=team_object,
                    llm_router=llm_router,
                ):
                    raise

    # 2.2. If team member has per-member model scope, enforce it
    if _model and team_object and valid_token and valid_token.user_id:
        with tracer.trace("litellm.proxy.auth.common_checks.check_team_member_model_access"):
            await _check_team_member_model_access(
                model=_model,
                team_object=team_object,
                valid_token=valid_token,
                llm_router=llm_router,
                prisma_client=prisma_client,
                user_api_key_cache=user_api_key_cache,
                proxy_logging_obj=proxy_logging_obj,
            )

    # Require trace id for agent keys when agent has require_trace_id_on_calls_by_agent
    if valid_token is not None and valid_token.agent_id:
        from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
        from litellm.proxy.litellm_pre_call_utils import get_chain_id_from_headers

        agent: Final = global_agent_registry.get_agent_by_id(agent_id=valid_token.agent_id)
        if agent is not None:
            require_trace_id: Final = (agent.litellm_params or {}).get("require_trace_id_on_calls_by_agent")
            if require_trace_id:
                headers_dict: Final = dict(request.headers)
                trace_id: Final = get_chain_id_from_headers(headers_dict)
                if not trace_id:
                    raise ProxyException(
                        message="Requests made with this agent's key must include the x-litellm-trace-id header.",
                        type=ProxyErrorTypes.bad_request_error,
                        param=None,
                        code=status.HTTP_400_BAD_REQUEST,
                    )

    ## 2.1 If user can call model (if personal key)
    if _model and team_object is None and user_object is not None:
        with tracer.trace("litellm.proxy.auth.common_checks.can_user_call_model"):
            await can_user_call_model(
                model=_model,
                llm_router=llm_router,
                user_object=user_object,
            )

    # 1.1 - 2.2 - 3.0.2 - 3.0.3: Project checks (blocked, model access, budget)
    with tracer.trace("litellm.proxy.auth.common_checks.run_project_checks"):
        await _run_project_checks(
            project_object=project_object,
            _model=_model,
            llm_router=llm_router,
            skip_budget_checks=skip_all_budget_checks,
            valid_token=valid_token,
            proxy_logging_obj=proxy_logging_obj,
        )

    # Run before apply_key_tags_pre_auth injects key metadata.tags into request_body.
    _reject_clientside_metadata_tags_check(general_settings, request_body, route)

    # If this is a free model, skip all budget checks
    if not skip_all_budget_checks:
        # Key metadata.tags are injected into request_body here so the tag budget
        # check can read them; this mutation must run before the gathered checks.
        if valid_token is not None:
            from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup

            LiteLLMProxyRequestSetup.pre_seed_litellm_metadata_for_route(
                request_data=request_body,
                route=route,
            )

            LiteLLMProxyRequestSetup.apply_key_tags_pre_auth(
                request_data=request_body,
                user_api_key_dict=valid_token,
            )

        async def _user_max_budget_check() -> None:
            # 4.1 personal budget
            if user_object is None or user_object.max_budget is None:
                return
            is_team_key: Final = team_object is not None and team_object.team_id is not None
            if is_team_key and general_settings.get("apply_user_budget_to_team_keys") is not True:
                return

            from litellm.proxy.proxy_server import get_current_spend

            user_budget: Final = user_object.max_budget
            user_spend: Final = await get_current_spend(
                counter_key=f"spend:user:{user_object.user_id}",
                fallback_spend=user_object.spend or 0.0,
                max_budget=user_budget,
            )
            if math.isfinite(user_budget) and user_spend >= user_budget:
                raise litellm.BudgetExceededError(
                    current_cost=user_spend,
                    max_budget=user_budget,
                    message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}",
                    entity_type=Litellm_EntityType.USER.value,
                    entity_id=user_object.user_id,
                )

        # Each scope reads a distinct counter key with no cross-scope ordering
        # dependency, so the per-scope Redis-first reads run concurrently instead
        # of one sequential await per scope. return_exceptions lets every scope
        # settle, then the first error in scope-priority order propagates exactly
        # as the sequential path raised.
        budget_check_coros: Final = tuple(
            coro
            for coro in (
                _team_max_budget_check(
                    team_object=team_object,
                    proxy_logging_obj=proxy_logging_obj,
                    valid_token=valid_token,
                ),
                _team_multi_budget_check(team_object=team_object),
                _virtual_key_multi_budget_check(valid_token=valid_token) if valid_token is not None else None,
                _team_soft_budget_check(
                    team_object=team_object,
                    proxy_logging_obj=proxy_logging_obj,
                    valid_token=valid_token,
                ),
                _organization_max_budget_check(
                    valid_token=valid_token,
                    team_object=team_object,
                    prisma_client=prisma_client,
                    user_api_key_cache=user_api_key_cache,
                    proxy_logging_obj=proxy_logging_obj,
                ),
                _tag_max_budget_check(
                    request_body=request_body,
                    prisma_client=prisma_client,
                    user_api_key_cache=user_api_key_cache,
                    proxy_logging_obj=proxy_logging_obj,
                    valid_token=valid_token,
                ),
                _user_max_budget_check(),
                _check_team_member_budget(
                    team_object=team_object,
                    user_object=user_object,
                    valid_token=valid_token,
                    prisma_client=prisma_client,
                    user_api_key_cache=user_api_key_cache,
                    proxy_logging_obj=proxy_logging_obj,
                ),
                _check_end_user_budget(end_user_obj=end_user_object, route=route)
                if end_user_object is not None and end_user_object.litellm_budget_table is not None
                else None,
            )
            if coro is not None
        )

        with tracer.trace("litellm.proxy.auth.common_checks.budget_checks"):
            budget_results: Final = await asyncio.gather(*budget_check_coros, return_exceptions=True)
        budget_error: Final = next((r for r in budget_results if isinstance(r, BaseException)), None)
        if budget_error is not None:
            raise budget_error

    _enforce_user_param_check(general_settings, request, request_body, route)
    _global_proxy_budget_check(global_proxy_spend, skip_all_budget_checks, route)
    _guardrail_modification_check(_typed_request_body(request_body), team_object)

    # 10 [OPTIONAL] Organization RBAC checks
    organization_role_based_access_check(user_object=user_object, route=route, request_body=request_body)

    async def _fetch_team_org_id(team_id: str) -> str | None:
        try:
            team: Final = await get_team_object(
                team_id=team_id,
                prisma_client=prisma_client,
                user_api_key_cache=user_api_key_cache,
                proxy_logging_obj=proxy_logging_obj,
            )
        except HTTPException:
            return None
        return team.organization_id

    request_body_for_route_check: Final = await add_team_org_context_to_request_body(
        route=route,
        request_body=request_body,
        fetch_team_org_id=_fetch_team_org_id,
        route_template=get_request_route_template(request),
    )

    _is_route_allowed: Final = _is_api_route_allowed(
        route=route,
        request=request,
        request_data=request_body_for_route_check,
        valid_token=valid_token,
        user_obj=user_object,
    )

    # 11. [OPTIONAL] Vector store checks - is the object allowed to access the vector store
    with tracer.trace("litellm.proxy.auth.common_checks.vector_store_access_check"):
        await vector_store_access_check(
            request_body=request_body,
            team_object=team_object,
            valid_token=valid_token,
        )

    # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path)
    with tracer.trace("litellm.proxy.auth.common_checks.check_tools_allowlist"):
        await check_tools_allowlist(
            request_body=request_body,
            valid_token=valid_token,
            team_object=team_object,
            route=route,
        )

    return True


def _get_user_role(
    user_obj: LiteLLM_UserTable | None,
) -> LitellmUserRoles | None:
    if user_obj is None:
        return None

    _user: Final = user_obj

    _user_role: Final = _user.user_role
    try:
        role: Final = LitellmUserRoles(_user_role)
    except ValueError:
        return LitellmUserRoles.INTERNAL_USER

    return role


def _is_api_route_allowed(
    route: str,
    request: Request,
    request_data: dict,
    valid_token: UserAPIKeyAuth | None,
    user_obj: LiteLLM_UserTable | None = None,
) -> bool:
    """
    - Route b/w api token check and normal token check
    """
    _user_role: Final = _get_user_role(user_obj=user_obj)

    if valid_token is None:
        raise Exception("Invalid proxy server token passed. valid_token=None.")

    if not _is_user_proxy_admin(user_obj=user_obj):  # if non-admin
        RouteChecks.non_proxy_admin_allowed_routes_check(
            user_obj=user_obj,
            _user_role=_user_role,
            route=route,
            request=request,
            request_data=request_data,
            valid_token=valid_token,
        )
    return True


def _is_user_proxy_admin(user_obj: LiteLLM_UserTable | None):
    if user_obj is None:
        return False

    if user_obj.user_role is not None and user_obj.user_role == LitellmUserRoles.PROXY_ADMIN.value:
        return True

    return False


def _allowed_routes_check(user_route: str, allowed_routes: list) -> bool:
    """
    Return if a user is allowed to access route. Helper function for `allowed_routes_check`.

    Parameters:
    - user_route: str - the route the user is trying to call
    - allowed_routes: List[str|LiteLLMRoutes] - the list of allowed routes for the user.
    """
    from starlette.routing import compile_path

    for allowed_route in allowed_routes:
        if allowed_route in LiteLLMRoutes.__members__:
            for template in LiteLLMRoutes[allowed_route].value:
                regex, _, _ = compile_path(template)
                if regex.match(user_route):
                    return True
        elif allowed_route == user_route:
            return True
    return False


def allowed_routes_check(
    user_role: LitellmUserRoles,
    user_route: str,
    litellm_proxy_roles: LiteLLM_JWTAuth,
) -> bool:
    """
    Check if user -> not admin - allowed to access these routes
    """

    if user_role == LitellmUserRoles.PROXY_ADMIN:
        is_allowed = _allowed_routes_check(
            user_route=user_route,
            allowed_routes=litellm_proxy_roles.admin_allowed_routes,
        )
        return is_allowed

    elif user_role == LitellmUserRoles.TEAM:
        if litellm_proxy_roles.team_allowed_routes is None:
            """
            By default allow a team to call openai + info routes
            """
            is_allowed = _allowed_routes_check(user_route=user_route, allowed_routes=["openai_routes", "info_routes"])
            return is_allowed
        elif litellm_proxy_roles.team_allowed_routes is not None:
            is_allowed = _allowed_routes_check(
                user_route=user_route,
                allowed_routes=litellm_proxy_roles.team_allowed_routes,
            )
            return is_allowed
    return False


def allowed_route_check_inside_route(
    user_api_key_dict: UserAPIKeyAuth,
    requested_user_id: str | None,
) -> bool:
    ret_val = True
    if (
        user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN
        and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY
    ):
        ret_val = False
    if requested_user_id is not None and user_api_key_dict.user_id is not None:
        if user_api_key_dict.user_id == requested_user_id:
            ret_val = True
    return ret_val


def get_actual_routes(allowed_routes: list) -> list:
    actual_routes: Final[list] = []
    for route_name in allowed_routes:
        try:
            route_value = LiteLLMRoutes[route_name].value
            if isinstance(route_value, set):
                actual_routes.extend(list(route_value))
            else:
                actual_routes.extend(route_value)

        except KeyError:
            actual_routes.append(route_name)
    return actual_routes


async def get_default_end_user_budget(
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
) -> LiteLLM_BudgetTable | None:
    """
    Fetches the default end user budget from the database if litellm.max_end_user_budget_id is configured.

    This budget is applied to end users who don't have an explicit budget_id set.
    Results are cached for performance.

    Args:
        prisma_client: Database client instance
        user_api_key_cache: Cache for storing/retrieving budget data
        parent_otel_span: Optional OpenTelemetry span for tracing

    Returns:
        LiteLLM_BudgetTable if configured and found, None otherwise
    """
    if prisma_client is None or litellm.max_end_user_budget_id is None:
        return None

    cache_key: Final = f"default_end_user_budget:{litellm.max_end_user_budget_id}"

    # Check cache first
    cached_budget: Final = await user_api_key_cache.async_get_cache(
        key=cache_key,
        model_type=LiteLLM_BudgetTable,
    )
    if cached_budget is not None:
        return cached_budget

    # Fetch from database
    try:
        budget_record: Final = await _dictable_table(BudgetRepository(prisma_client)).find_unique(
            where={"budget_id": litellm.max_end_user_budget_id}
        )

        if budget_record is None:
            verbose_proxy_logger.warning(
                "Default end user budget not found in database: %s", litellm.max_end_user_budget_id
            )
            return None

        _budget_obj: Final = LiteLLM_BudgetTable.model_validate(budget_record.dict())
        # Cache the budget for 60 seconds
        await user_api_key_cache.async_set_cache(
            key=cache_key,
            value=_budget_obj,
            model_type=LiteLLM_BudgetTable,
            ttl=get_management_object_ttl(user_api_key_cache),
        )

        return _budget_obj

    except Exception as e:
        verbose_proxy_logger.error("Error fetching default end user budget: %s", e)
        return None


@log_db_metrics
async def get_team_member_default_budget(
    budget_id: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
) -> LiteLLM_BudgetTable | None:
    """
    Fetches the team-level default per-member budget referenced by team.metadata["team_member_budget_id"].

    This budget is applied to team members whose TeamMembership row has no
    linked budget, or whose linked budget has max_budget=NULL. Results are
    cached for performance.

    Args:
        budget_id: The budget_id pulled from team.metadata["team_member_budget_id"]
        prisma_client: Database client instance
        user_api_key_cache: Cache for storing/retrieving budget data

    Returns:
        LiteLLM_BudgetTable if found, None otherwise
    """
    if prisma_client is None:
        return None

    cache_key: Final = f"team_member_default_budget:{budget_id}"

    cached_budget: Final = await user_api_key_cache.async_get_cache(
        key=cache_key,
        model_type=LiteLLM_BudgetTable,
    )
    if cached_budget is not None:
        return cached_budget

    try:
        budget_record: Final = await _dictable_table(BudgetRepository(prisma_client)).find_unique(
            where={"budget_id": budget_id}
        )
    except Exception:
        verbose_proxy_logger.exception("Error fetching team-default member budget %s", budget_id)
        return None

    if budget_record is None:
        verbose_proxy_logger.warning("Team-default member budget not found in database: %s", budget_id)
        return None

    budget: Final = LiteLLM_BudgetTable.model_validate(budget_record.dict())
    await user_api_key_cache.async_set_cache(
        key=cache_key,
        value=budget,
        model_type=LiteLLM_BudgetTable,
        ttl=get_management_object_ttl(user_api_key_cache),
    )

    return budget


async def _apply_default_budget_to_end_user(
    end_user_obj: LiteLLM_EndUserTable,
    prisma_client: PrismaClient,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
) -> LiteLLM_EndUserTable:
    """
    Helper function to apply default budget to end user if they don't have a budget assigned.

    Args:
        end_user_obj: The end user object to potentially apply default budget to
        prisma_client: Database client instance
        user_api_key_cache: Cache for storing/retrieving data
        parent_otel_span: Optional OpenTelemetry span for tracing

    Returns:
        Updated end user object with default budget applied if applicable
    """
    # If end user already has a budget assigned, no need to apply default
    if end_user_obj.litellm_budget_table is not None:
        return end_user_obj

    # If no default budget configured, return as-is
    if litellm.max_end_user_budget_id is None:
        return end_user_obj

    # Fetch and apply default budget
    default_budget: Final = await get_default_end_user_budget(
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        parent_otel_span=parent_otel_span,
    )

    if default_budget is not None:
        # Apply default budget to end user object
        end_user_obj.litellm_budget_table = default_budget
        verbose_proxy_logger.debug(
            "Applied default budget %s to end user %s", litellm.max_end_user_budget_id, end_user_obj.user_id
        )

    return end_user_obj


async def _check_end_user_budget(
    end_user_obj: LiteLLM_EndUserTable,
    route: str,
) -> None:
    """
    Check if end user is within their budget limit.

    Args:
        end_user_obj: The end user object to check
        route: The request route

    Raises:
        litellm.BudgetExceededError: If end user has exceeded their budget
    """
    if RouteChecks.is_info_route(route):
        return

    if end_user_obj.litellm_budget_table is None:
        return

    end_user_budget: Final = end_user_obj.litellm_budget_table.max_budget
    if end_user_budget is None:
        return

    from litellm.proxy.proxy_server import get_current_spend

    end_user_spend: Final = await get_current_spend(
        counter_key=f"spend:end_user:{end_user_obj.user_id}",
        fallback_spend=end_user_obj.spend or 0.0,
        max_budget=end_user_budget,
        fallback_authoritative=True,
    )
    if end_user_spend > end_user_budget:
        raise litellm.BudgetExceededError(
            current_cost=end_user_spend,
            max_budget=end_user_budget,
            message=f"ExceededBudget: End User={end_user_obj.user_id} over budget. Spend={end_user_spend}, Budget={end_user_budget}",
            entity_type=Litellm_EntityType.END_USER.value,
            entity_id=end_user_obj.user_id,
        )


#: Columns whose non-null value makes an end-user row restrict something auth enforces. ``blocked``
#: is separate: it restricts when true rather than when merely set.
_RESTRICTED_COLUMNS: Final = ("budget_id", "allowed_model_region", "default_model", "object_permission_id")


def _column_is_set(column: str) -> Mapping[str, object]:
    """``column IS NOT NULL`` as a plain dict, which is the only shape prisma's builder accepts."""
    return {column: {"not": None}}  # mutable-ok: prisma's query builder isinstance-checks for dict


def _restricted_end_user_where() -> Mapping[str, object]:
    """Prisma filter selecting every end-user row that carries a restriction auth enforces."""
    return {"OR": [{"blocked": True}, *map(_column_is_set, _RESTRICTED_COLUMNS)]}  # mutable-ok: prisma needs dict/list


class _RegistryNotCached:
    """No cached registry answer, as distinct from the cached answer ``None`` (registry unusable)."""


_REGISTRY_NOT_CACHED: Final = _RegistryNotCached()

#: One lock per registry; module-level because the stampede to collapse is worker-wide.
_TAG_REGISTRY_LOAD_LOCK: Final = asyncio.Lock()
_END_USER_REGISTRY_LOAD_LOCK: Final = asyncio.Lock()


async def _cached_registry(
    cache_key: str,
    overflow_sentinel: str,
    user_api_key_cache: UserApiKeyCache,
) -> frozenset[str] | None | _RegistryNotCached:
    """The cached registry answer, or ``_REGISTRY_NOT_CACHED`` when the caller has to query."""
    cached: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=cache_key)
    if cached == overflow_sentinel:
        return None
    # Memory hands back the tuple that was written; Redis round-trips it through JSON as a list.
    if isinstance(cached, (list, tuple)):
        return frozenset(entry for entry in cached if isinstance(entry, str))
    return _REGISTRY_NOT_CACHED


async def _cache_registry_answer(
    cache_key: str,
    value: tuple[str, ...] | str,
    ttl: float,
    user_api_key_cache: UserApiKeyCache,
) -> None:
    """Best-effort: a cache backend failure must not turn a registry load into a failed request."""
    try:
        await user_api_key_cache.async_set_cache(key=cache_key, value=value, ttl=ttl)
    except Exception as e:  # noqa: BLE001  # best-effort cache write: auth must survive a cache backend error
        verbose_proxy_logger.warning("Failed to cache registry %s: %s", cache_key, e)


async def _fetch_and_cache_registry(
    cache_key: str,
    overflow_sentinel: str,
    max_size: int,
    fetch_ids: Callable[[], Awaitable[tuple[str, ...]]],
    user_api_key_cache: UserApiKeyCache,
) -> frozenset[str] | None:
    """The registry as the database has it, cached whole, or ``None`` when it is unusable."""
    try:
        registry_ids: Final = await fetch_ids()
    except Exception as e:  # noqa: BLE001  # fail-safe: any registry load error must degrade to per-id lookups, never break auth
        verbose_proxy_logger.warning(
            "Registry %s could not be loaded from the database, so per-id lookups will run and the "
            "registry query is suppressed for %ss: %s",
            cache_key,
            REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
            e,
        )
        await _cache_registry_answer(
            cache_key=cache_key,
            value=overflow_sentinel,
            ttl=REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
            user_api_key_cache=user_api_key_cache,
        )
        return None

    if len(registry_ids) > max_size:
        await _cache_registry_answer(
            cache_key=cache_key,
            value=overflow_sentinel,
            ttl=get_management_object_ttl(user_api_key_cache),
            user_api_key_cache=user_api_key_cache,
        )
        return None

    await _cache_registry_answer(
        cache_key=cache_key,
        value=registry_ids,
        ttl=get_management_object_ttl(user_api_key_cache),
        user_api_key_cache=user_api_key_cache,
    )
    return frozenset(registry_ids)


async def _load_bounded_registry(
    cache_key: str,
    overflow_sentinel: str,
    max_size: int,
    load_lock: asyncio.Lock,
    fetch_ids: Callable[[], Awaitable[tuple[str, ...]]],
    user_api_key_cache: UserApiKeyCache,
) -> frozenset[str] | None:
    """
    A bounded id set under one cache key, so an id outside it costs no DB read.

    ``None`` = unusable (overflow or recent DB error): fall back to per-id lookups. An empty
    frozenset is a real, cacheable answer. Loads are single-flighted to stop TTL-expiry stampedes.
    """
    cached: Final = await _cached_registry(cache_key, overflow_sentinel, user_api_key_cache)
    if not isinstance(cached, _RegistryNotCached):
        return cached

    async with load_lock:
        # The request that held the lock has since cached an answer for everyone waiting on it.
        cached_after_wait: Final = await _cached_registry(cache_key, overflow_sentinel, user_api_key_cache)
        if not isinstance(cached_after_wait, _RegistryNotCached):
            return cached_after_wait

        return await _fetch_and_cache_registry(
            cache_key=cache_key,
            overflow_sentinel=overflow_sentinel,
            max_size=max_size,
            fetch_ids=fetch_ids,
            user_api_key_cache=user_api_key_cache,
        )


async def _load_end_user_restricted_registry(
    prisma_client: PrismaClient,
    user_api_key_cache: UserApiKeyCache,
) -> frozenset[str] | None:
    """The set of end-user ids whose ``LiteLLM_EndUserTable`` row carries a restriction."""

    async def fetch_ids() -> tuple[str, ...]:
        restricted_rows: Final = await _end_user_table(EndUserRepository(prisma_client)).find_many(
            where=_restricted_end_user_where(),
            take=END_USER_RESTRICTED_REGISTRY_MAX_SIZE + 1,
        )
        return tuple(row.user_id for row in restricted_rows)

    return await _load_bounded_registry(
        cache_key=end_user_restricted_registry_cache_key(),
        overflow_sentinel=END_USER_RESTRICTED_REGISTRY_OVERFLOW_SENTINEL,
        max_size=END_USER_RESTRICTED_REGISTRY_MAX_SIZE,
        load_lock=_END_USER_REGISTRY_LOAD_LOCK,
        fetch_ids=fetch_ids,
        user_api_key_cache=user_api_key_cache,
    )


async def _end_user_is_known_unrestricted(
    end_user_id: str,
    prisma_client: PrismaClient,
    user_api_key_cache: UserApiKeyCache,
    token_end_user_max_budget: float | None,
) -> bool:
    """
    True when the cached registry proves the id restricts nothing, so its row need not be read.

    Every field ``get_end_user_object`` callers consume (budget, spend under that budget, region,
    default model, object permission, blocked) is part of the registry predicate, so an id outside
    it is indistinguishable from one with no row at all. The skip is off whenever mere existence of
    the row is meaningful: ``max_end_user_budget_id`` grafts a default budget onto any row that
    exists, ``validate_end_user_id_in_db`` rejects ids that resolve to no row, and a token-supplied
    ``end_user_max_budget`` (a ``user_custom_auth`` callable can set one against an otherwise
    unrestricted row) is enforced against the row's recorded spend.
    """
    if (
        litellm.max_end_user_budget_id is not None
        or litellm.validate_end_user_id_in_db
        or token_end_user_max_budget is not None
    ):
        return False

    registry: Final = await _load_end_user_restricted_registry(
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
    )
    return registry is not None and end_user_id not in registry


@log_db_metrics
async def get_end_user_object(
    end_user_id: str | None,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    route: str | None = "",
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
    token_end_user_max_budget: float | None = None,
) -> LiteLLM_EndUserTable | None:
    """
    Returns end user object from database or cache.

    If end user exists but has no budget_id, applies the default budget
    (if configured via litellm.max_end_user_budget_id).

    Args:
        end_user_id: The ID of the end user
        prisma_client: Database client instance
        user_api_key_cache: Cache for storing/retrieving data
        route: The request route
        parent_otel_span: Optional OpenTelemetry span for tracing
        proxy_logging_obj: Optional proxy logging object
        token_end_user_max_budget: ``valid_token.end_user_max_budget``, when the caller holds a
            token. Budget enforcement reads the row's spend, so a row that restricts nothing on
            its own must still be loaded when the token carries a budget for it.

    Returns:
        LiteLLM_EndUserTable if found, None otherwise
    """
    if prisma_client is None:
        raise Exception("No db connected")

    if end_user_id is None:
        return None

    _key: Final = end_user_cache_key(end_user_id)

    # Check cache first
    cached_user_obj: Final = await user_api_key_cache.async_get_cache(
        key=_key,
        model_type=LiteLLM_EndUserTable,
    )
    if cached_user_obj is not None:
        return_obj = cached_user_obj
        # Apply default budget if needed
        return_obj = await _apply_default_budget_to_end_user(
            end_user_obj=return_obj,
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            parent_otel_span=parent_otel_span,
        )

        return return_obj

    if await _end_user_is_known_unrestricted(
        end_user_id=end_user_id,
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        token_end_user_max_budget=token_end_user_max_budget,
    ):
        return None

    # Fetch from database
    try:
        response: Final = await _dictable_table(EndUserRepository(prisma_client)).find_unique(
            where={"user_id": end_user_id},
            include={"litellm_budget_table": True, "object_permission": True},
        )

        if response is None:
            raise Exception

        # Convert to LiteLLM_EndUserTable object
        _response = LiteLLM_EndUserTable.model_validate(response.dict())

        # Apply default budget if needed
        _response = await _apply_default_budget_to_end_user(
            end_user_obj=_response,
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            parent_otel_span=parent_otel_span,
        )

        # Save to cache
        await user_api_key_cache.async_set_cache(
            key=_key,
            value=_response,
            model_type=LiteLLM_EndUserTable,
            ttl=get_management_object_ttl(user_api_key_cache),
        )

        return _response

    except Exception:
        return None


_END_USER_VALIDATION_NEGATIVE_TTL: Final = 60
_END_USER_VALIDATION_POSITIVE_TTL: Final = 300


async def resolve_and_validate_end_user_id(
    raw_end_user_id: str | None,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
    route: str = "",
) -> str | None:
    """Optionally drop end-user ids that don't resolve to a known DB row.

    Default: pass-through. LiteLLM's documented pattern is that the `user`
    field is an arbitrary caller-supplied identifier, so validation is
    opt-in behind ``litellm.validate_end_user_id_in_db`` to preserve
    backwards compatibility.

    When the flag is set: accept the id when it matches any of
      - LiteLLM_EndUserTable.user_id
      - LiteLLM_UserTable.user_id
      - LiteLLM_UserTable.user_email (case-insensitive)

    If the id doesn't match but ``litellm.max_end_user_budget_id`` is set,
    we still preserve the id so the default end-user budget is applied
    downstream; otherwise we return None.

    DB lookups reuse ``get_end_user_object`` / ``get_user_object`` so they
    share the same cache as the rest of the auth path instead of adding new
    raw Prisma queries.
    """
    if raw_end_user_id is None:
        return None
    if not litellm.validate_end_user_id_in_db:
        return raw_end_user_id
    if prisma_client is None:
        return raw_end_user_id

    cache_key: Final = f"end_user_validation:{raw_end_user_id}"
    cached: Final = await _raw_cache(user_api_key_cache).async_get_cache(key=cache_key)
    if cached == "valid":
        return raw_end_user_id
    if cached == "invalid":
        return raw_end_user_id if litellm.max_end_user_budget_id else None

    is_valid: Final = await _end_user_id_exists_in_db(
        end_user_id=raw_end_user_id,
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        parent_otel_span=parent_otel_span,
        proxy_logging_obj=proxy_logging_obj,
        route=route,
    )

    await user_api_key_cache.async_set_cache(
        key=cache_key,
        value="valid" if is_valid else "invalid",
        ttl=(_END_USER_VALIDATION_POSITIVE_TTL if is_valid else _END_USER_VALIDATION_NEGATIVE_TTL),
    )

    if is_valid:
        return raw_end_user_id
    # Preserve id so the caller can still apply litellm.max_end_user_budget_id.
    if litellm.max_end_user_budget_id:
        return raw_end_user_id
    return None


async def _end_user_id_exists_in_db(
    end_user_id: str,
    prisma_client: PrismaClient,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
    route: str = "",
) -> bool:
    """True when the id matches an EndUser, User, or user_email row."""
    try:
        end_user_obj: Final = await get_end_user_object(
            end_user_id=end_user_id,
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            parent_otel_span=parent_otel_span,
            proxy_logging_obj=proxy_logging_obj,
            route=route,
        )
        if end_user_obj is not None:
            return True
    except Exception as e:
        verbose_proxy_logger.debug("end_user validation: get_end_user_object lookup failed: %s", e)

    try:
        user_obj: Final = await get_user_object(
            user_id=end_user_id,
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            user_id_upsert=False,
            parent_otel_span=parent_otel_span,
            proxy_logging_obj=proxy_logging_obj,
            check_db_only=False,
            user_email=end_user_id if "@" in end_user_id else None,
        )
        if user_obj is not None:
            return True
    except Exception as e:
        verbose_proxy_logger.debug("end_user validation: get_user_object lookup failed: %s", e)

    return False


async def _load_tag_registry(
    prisma_client: PrismaClient,
    user_api_key_cache: UserApiKeyCache,
) -> frozenset[str] | None:
    """The set of tag names that have a row in ``LiteLLM_TagTable``."""

    async def fetch_ids() -> tuple[str, ...]:
        registry_rows: Final = await _tag_table(TagRepository(prisma_client)).find_many(
            take=TAG_REGISTRY_MAX_SIZE + 1,
        )
        return tuple(row.tag_name for row in registry_rows)

    return await _load_bounded_registry(
        cache_key=tag_registry_cache_key(),
        overflow_sentinel=TAG_REGISTRY_OVERFLOW_SENTINEL,
        max_size=TAG_REGISTRY_MAX_SIZE,
        load_lock=_TAG_REGISTRY_LOAD_LOCK,
        fetch_ids=fetch_ids,
        user_api_key_cache=user_api_key_cache,
    )


async def _fetch_uncached_tags(
    uncached_tags: Sequence[str],
    prisma_client: PrismaClient,
    user_api_key_cache: UserApiKeyCache,
) -> tuple[tuple[str, LiteLLM_TagTable], ...]:
    """Rows for the tags a cache probe missed; names absent from the registry never reach the DB."""
    if not uncached_tags:
        return ()

    registry: Final = await _load_tag_registry(
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
    )
    tags_to_fetch: Final = (
        tuple(uncached_tags) if registry is None else tuple(tag for tag in uncached_tags if tag in registry)
    )
    if not tags_to_fetch:
        return ()

    try:
        db_tags: Final = await _tag_table(TagRepository(prisma_client)).find_many(
            where={"tag_name": {"in": list(tags_to_fetch)}},
            include={"litellm_budget_table": True},
        )
        fetched: Final = tuple((db_tag.tag_name, LiteLLM_TagTable.model_validate(db_tag.dict())) for db_tag in db_tags)
        for fetched_name, fetched_obj in fetched:
            await user_api_key_cache.async_set_cache(
                key=tag_cache_key(fetched_name),
                value=fetched_obj,
                model_type=LiteLLM_TagTable,
                ttl=get_management_object_ttl(user_api_key_cache),
            )
    except Exception as e:  # noqa: BLE001  # fail-safe: a tag fetch error must yield "no budget objects", never break auth
        verbose_proxy_logger.debug("Error batch fetching tags from database: %s", e)
        return ()
    else:
        return fetched


@log_db_metrics
async def get_tag_objects_batch(
    tag_names: list[str],
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> dict[str, LiteLLM_TagTable]:
    """
    Batch fetch multiple tag objects from cache and db.

    Optimizes for latency by:
    1. Serving already-cached tags without touching the DB
    2. Skipping tags that no ``LiteLLM_TagTable`` row exists for, via the cached name registry
    3. Batch fetching the remaining uncached tags in one DB query

    Args:
        tag_names: List of tag names to fetch
        prisma_client: Prisma database client
        user_api_key_cache: Cache for storing tag objects
        parent_otel_span: Optional OpenTelemetry span for tracing
        proxy_logging_obj: Optional proxy logging object

    Returns:
        Dictionary mapping tag_name to LiteLLM_TagTable object
    """
    if prisma_client is None or not tag_names:
        return {}

    probed: Final = [
        (
            tag_name,
            await user_api_key_cache.async_get_cache(key=tag_cache_key(tag_name), model_type=LiteLLM_TagTable),
        )
        for tag_name in tag_names
    ]
    fetched: Final = await _fetch_uncached_tags(
        uncached_tags=tuple(tag_name for tag_name, tag_obj in probed if tag_obj is None),
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
    )
    return {tag_name: tag_obj for tag_name, tag_obj in (*probed, *fetched) if tag_obj is not None}


@log_db_metrics
async def get_tag_object(
    tag_name: str | None,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> LiteLLM_TagTable | None:
    """
    Returns tag object from cache or db.

    Uses default cache TTL (same as end_user objects) to avoid drift.

    Args:
        tag_name: Name of the tag to fetch
        prisma_client: Prisma database client
        user_api_key_cache: Cache for storing tag objects
        parent_otel_span: Optional OpenTelemetry span for tracing
        proxy_logging_obj: Optional proxy logging object

    Returns:
        LiteLLM_TagTable object if found, None otherwise
    """
    if prisma_client is None or tag_name is None:
        return None

    # Use batch helper for consistency
    tag_objects: Final = await get_tag_objects_batch(
        tag_names=[tag_name],
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        parent_otel_span=parent_otel_span,
        proxy_logging_obj=proxy_logging_obj,
    )

    return tag_objects.get(tag_name)


@log_db_metrics
async def get_team_membership(
    user_id: str,
    team_id: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> Optional["LiteLLM_TeamMembership"]:
    """
    Returns team membership object if user is member of team.

    Do a isolated check for team membership vs. doing a combined key + team + user + team-membership check, as key might come in frequently for different users/teams. Larger call will slowdown query time. This way we get to cache the constant (key/team/user info) and only update based on the changing value (team membership).
    """
    from litellm.proxy._types import LiteLLM_TeamMembership

    if prisma_client is None:
        raise Exception("No db connected")

    if user_id is None or team_id is None:
        return None

    _key: Final = team_membership_reservation_cache_key(user_id=user_id, team_id=team_id)

    # check if in cache
    cached_membership_obj: Final = await user_api_key_cache.async_get_cache(
        key=_key,
        model_type=LiteLLM_TeamMembership,
    )
    if cached_membership_obj is not None:
        return cached_membership_obj

    # else, check db
    try:
        response: Final = await _dictable_table(TeamMembershipRepository(prisma_client)).find_unique(
            where={"user_id_team_id": {"user_id": user_id, "team_id": team_id}},
            include={"litellm_budget_table": True},
        )

        if response is None:
            return None

        _response: Final = LiteLLM_TeamMembership.model_validate(response.dict())
        await user_api_key_cache.async_set_cache(
            key=_key,
            value=_response,
            model_type=LiteLLM_TeamMembership,
        )

        return _response
    except Exception:
        verbose_proxy_logger.exception(
            "Error getting team membership for user_id: %s, team_id: %s",
            user_id,
            team_id,
        )
        return None


def model_in_access_group(model: str, team_models: list[str] | None, llm_router: Router | None) -> bool:
    from collections import defaultdict

    if team_models is None:
        return True
    if model in team_models:
        return True

    access_groups: dict[str, list[str]] = defaultdict(list)
    if llm_router:
        access_groups = llm_router.get_model_access_groups(model_name=model)

    if len(access_groups) > 0:  # check if token contains any model access groups
        for idx, m in enumerate(
            team_models
        ):  # loop token models, if any of them are an access group add the access group
            if m in access_groups:
                return True

    # Filter out models that are access_groups
    filtered_models: Final = [m for m in team_models if m not in access_groups]

    if model in filtered_models:
        return True

    return False


def _should_check_db(key: str, last_db_access_time: LimitedSizeOrderedDict, db_cache_expiry: int) -> bool:
    """
    Prevent calling db repeatedly for items that don't exist in the db.
    """
    current_time: Final = time.time()
    # if key doesn't exist in last_db_access_time -> check db
    if key not in last_db_access_time or last_db_access_time[key][0] is not None:
        return True
    elif last_db_access_time[key][0] is None:
        if current_time - last_db_access_time[key][1] >= db_cache_expiry:
            return True
    return False


def _update_last_db_access_time(key: str, value: object | None, last_db_access_time: LimitedSizeOrderedDict):
    last_db_access_time[key] = (value, time.time())


def _get_role_based_permissions(
    rbac_role: RBAC_ROLES,
    general_settings: dict,
    key: Literal["models", "routes"],
) -> list[str] | None:
    """
    Get the role based permissions from the general settings.
    """
    role_based_permissions: Final = cast(
        list[RoleBasedPermissions] | None,
        general_settings.get("role_permissions", []),
    )
    if role_based_permissions is None:
        return None

    for role_based_permission in role_based_permissions:
        if role_based_permission.role == rbac_role:
            return role_based_permission.models if key == "models" else role_based_permission.routes

    return None


def get_role_based_models(
    rbac_role: RBAC_ROLES,
    general_settings: dict,
) -> list[str] | None:
    """
    Get the models allowed for a user role.

    Used by JWT Auth.
    """

    return _get_role_based_permissions(
        rbac_role=rbac_role,
        general_settings=general_settings,
        key="models",
    )


def get_role_based_routes(
    rbac_role: RBAC_ROLES,
    general_settings: dict,
) -> list[str] | None:
    """
    Get the routes allowed for a user role.
    """

    return _get_role_based_permissions(
        rbac_role=rbac_role,
        general_settings=general_settings,
        key="routes",
    )


async def _get_fuzzy_user_object(
    prisma_client: PrismaClient,
    sso_user_id: str | None = None,
    user_email: str | None = None,
) -> "_PrismaUserRow | None":
    """
    Checks if sso user is in db.

    Called when user id match is not found in db.

    - Check if sso_user_id is user_id in db
    - Check if sso_user_id is sso_user_id in db
    - Check if user_email is user_email in db
    - If not, create new user with user_email and sso_user_id and user_id = sso_user_id
    """

    response = None
    if sso_user_id is not None:
        response = await _user_table(UserRepository(prisma_client)).find_unique(
            where={"sso_user_id": sso_user_id},
            include={"organization_memberships": True},
        )

    if response is None and user_email is not None:
        # Use case-insensitive query to handle emails with different casing
        # This matches the pattern used in _check_duplicate_user_email
        response = await _user_table(UserRepository(prisma_client)).find_first(
            where={"user_email": {"equals": user_email, "mode": "insensitive"}},
            include={"organization_memberships": True},
        )

        if response is not None and sso_user_id is not None:  # update sso_user_id
            asyncio.create_task(  # background task to update user with sso id
                _user_table(UserRepository(prisma_client)).update(
                    where={"user_id": response.user_id},
                    data={"sso_user_id": sso_user_id},
                )
            )

    return response


async def _backfill_null_user_email(
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    user_row: LiteLLM_UserTable,
    user_email: str | None,
) -> LiteLLM_UserTable:
    if user_email is None or user_row.user_email is not None or prisma_client is None:
        return user_row

    user_repo: Final = UserRepository(prisma_client)
    await user_repo.backfill_null_user_email(
        user_id=user_row.user_id,
        user_email=user_email,
    )
    db_row: Final = await user_repo.find_by_id(user_row.user_id)
    if db_row is None:
        return user_row
    email_update: Final = {"user_email": db_row.user_email}  # mutable-ok: model_copy update payload is dict-shaped
    updated_row: Final = user_row.model_copy(update=email_update)
    await user_api_key_cache.async_set_cache(
        key=user_row.user_id,
        value=updated_row,
        model_type=LiteLLM_UserTable,
        ttl=get_management_object_ttl(user_api_key_cache),
    )
    return updated_row


@log_db_metrics
async def get_user_object(
    user_id: str | None,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    user_id_upsert: bool,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
    sso_user_id: str | None = None,
    user_email: str | None = None,
    check_db_only: bool | None = None,
) -> LiteLLM_UserTable | None:
    """
    - Check if user id in proxy User Table
    - if valid, return LiteLLM_UserTable object with defined limits
    - if not, then raise an error
    """

    if user_id is None:
        return None

    # check if in cache
    if not check_db_only:
        cached_user_obj: Final = await user_api_key_cache.async_get_cache(
            key=user_id,
            model_type=LiteLLM_UserTable,
        )
        if cached_user_obj is not None:
            return await _backfill_null_user_email(
                prisma_client=prisma_client,
                user_api_key_cache=user_api_key_cache,
                user_row=cached_user_obj,
                user_email=user_email,
            )
    # else, check db
    if prisma_client is None:
        raise Exception("No db connected")
    try:
        db_access_time_key: Final = f"user_id:{user_id}"
        should_check_db: Final = _should_check_db(
            key=db_access_time_key,
            last_db_access_time=last_db_access_time,
            db_cache_expiry=db_cache_expiry,
        )

        if should_check_db:
            response = await _user_table(UserRepository(prisma_client)).find_unique(
                where={"user_id": user_id}, include={"organization_memberships": True}
            )

            if response is None:
                response = await _get_fuzzy_user_object(
                    prisma_client=prisma_client,
                    sso_user_id=sso_user_id,
                    user_email=user_email,
                )

        else:
            response = None

        if response is None:
            if user_id_upsert:
                from litellm.proxy.management_endpoints.internal_user_endpoints import (
                    add_new_user_to_default_team,
                    check_if_default_team_set,
                )

                default_params: Final = litellm.default_internal_user_params or {}
                scalar_default_params: Final = {
                    key: value for key, value in default_params.items() if key not in ("teams", "available_teams")
                }
                new_user_params: Final[dict[str, Any]] = {
                    "user_id": user_id,
                    **({"user_email": user_email} if user_email is not None else {}),
                    **scalar_default_params,
                }
                if (
                    new_user_params.get("budget_duration") is not None
                    and new_user_params.get("budget_reset_at") is None
                ):
                    new_user_params["budget_reset_at"] = get_budget_reset_time(
                        budget_duration=new_user_params["budget_duration"]
                    )

                response = await _user_table(UserRepository(prisma_client)).create(
                    data=new_user_params,
                    include={"organization_memberships": True},
                )

                default_teams: Final = check_if_default_team_set()
                if default_teams:
                    await add_new_user_to_default_team(
                        user_id=user_id,
                        user_email=user_email,
                        user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
                        teams=default_teams,
                        prisma_client=prisma_client,
                    )
            else:
                if should_check_db:
                    _update_last_db_access_time(
                        key=db_access_time_key,
                        value=None,
                        last_db_access_time=last_db_access_time,
                    )
                raise Exception

        if response.organization_memberships is not None and len(response.organization_memberships) > 0:
            # dump each organization membership to type LiteLLM_OrganizationMembershipTable
            _dumped_memberships: Final = [
                LiteLLM_OrganizationMembershipTable.model_validate(membership.model_dump())
                for membership in response.organization_memberships
                if membership is not None
            ]
            response.organization_memberships = _dumped_memberships

        _response = LiteLLM_UserTable.model_validate(dict(response))
        _response = await _backfill_null_user_email(
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            user_row=_response,
            user_email=user_email,
        )
        response_dict: Final = _response.model_dump()

        # save the user object to cache
        await user_api_key_cache.async_set_cache(
            key=user_id,
            value=_response,
            model_type=LiteLLM_UserTable,
            ttl=get_management_object_ttl(user_api_key_cache),
        )

        # save to db access time
        _update_last_db_access_time(
            key=db_access_time_key,
            value=response_dict,
            last_db_access_time=last_db_access_time,
        )

        return _response
    except Exception as e:  # if user not in db
        _log_budget_lookup_failure("user", e)
        raise ValueError(
            f"User doesn't exist in db. 'user_id'={user_id}. Create user via `/user/new` call. Got error - {e}"
        )


async def _cache_management_object(
    key: str,
    value: BaseModel | Mapping[str, object],
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None,
    *,
    model_type: type[BaseModel],
):
    """
    Persist management objects via ``UserApiKeyCache`` (in-memory + optional Redis).

    ``UserApiKeyCache`` serializes with ``model_type`` so Redis and in-memory stay aligned.
    """
    await user_api_key_cache.async_set_cache(
        key=key,
        value=value,
        model_type=model_type,
        ttl=get_management_object_ttl(user_api_key_cache),
    )


async def _cache_team_object(
    team_id: str,
    team_table: LiteLLM_TeamTableCachedObj,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None,
):
    ## CACHE REFRESH TIME!
    team_table.last_refreshed_at = time.time()

    key: Final = f"team_id:{team_id}"

    if proxy_logging_obj is not None:
        try:
            await proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache(key=key)
        except Exception as e:  # noqa: BLE001  # best-effort invalidation: any cache backend error must not fail the write
            verbose_proxy_logger.warning(
                "Failed to invalidate internal usage cache entry %s; "
                "a stale team object may be served until its TTL expires: %s",
                key,
                e,
            )

    # team_id is the table primary key — guaranteed unique, safe to write.
    await _cache_management_object(
        key=key,
        value=team_table,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
        model_type=LiteLLM_TeamTableCachedObj,
    )

    # Invalidate the alias-keyed cache so the JWT auth path with
    # `team_alias_jwt_field` (which reads via `get_team_object_by_alias`)
    # doesn't keep serving the pre-mutation team after every team-write
    # endpoint (team_model_add, team_model_delete, update_team, etc.).
    #
    # Why DELETE and not WRITE: `team_alias` has no UNIQUE constraint in
    # schema.prisma. Writing this cache from the generic refresh path
    # would let a team admin who renamed their team to collide with
    # another team's alias silently overwrite the cached team for
    # JWT-by-alias auth (veria-ai review on #28739). Deleting forces the
    # next reader through `get_team_object_by_alias`, which DOES enforce
    # uniqueness (len(teams) > 1 raises HTTPException) before populating
    # the cache from a verified single row.
    if team_table.team_alias:
        alias_key: Final = f"team_alias:{team_table.team_alias}"
        try:
            user_api_key_cache.delete_cache(key=alias_key)
            if proxy_logging_obj is not None:
                await proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache(key=alias_key)
        except Exception as e:  # noqa: BLE001  # best-effort invalidation: any cache backend error must not fail the mutation
            verbose_proxy_logger.warning(
                "Failed to invalidate cached team alias entry %s; "
                "a stale team object may be served until its TTL expires: %s",
                alias_key,
                e,
            )


async def invalidate_team_member_spend_state(
    user_id: str,
    team_id: str,
    user_api_key_cache: UserApiKeyCache,
    new_spend: float | None = None,
) -> None:
    """
    Clear every cached read path for one team member's budget so a spend
    reset or a raised cap takes effect on the next request instead of
    waiting on the membership cache's TTL.

    Two independently-keyed cache entries hold the same LiteLLM_TeamMembership
    row: user_api_key_auth.py's admission check writes ``{team_id}_{user_id}``,
    while budget_reservation.py's pre-call reservation and auth_checks.py's own
    get_team_membership() (used by _check_team_member_budget) both write
    ``team_membership:{user_id}:{team_id}``. Both formats must be invalidated
    explicitly; writing one does not refresh the other. All keys are also
    broadcast (LIT-3803): each worker's own in-memory copy (membership object,
    spend counter, or the counter's own short-TTL DB-floor marker) survives
    eviction elsewhere until its TTL, so the handling worker alone clearing its
    copy leaves every other worker still enforcing the pre-reset budget.

    ``new_spend`` is only passed by reset_team_member_spend_fn, which knows the
    exact post-reset value: it is SET everywhere (matching /key/{key}/reset_spend's
    own precedent) rather than deleted, so a worker's next read reflects it
    directly instead of re-deriving it through a DB reseed. team_member_update
    only changes the budget cap, not the tracked spend, so it passes no
    new_spend; the live spend counter is untouched in that case (deleting it
    would force a reseed from the DB's own spend column, which lags the live
    counter via periodic batch writes, briefly under-enforcing the raised cap
    against a spend value lower than what was actually tracked) and only the
    membership caches carrying the new cap are invalidated.

    The floor marker (``spend_db_floor:``, proxy_server.py's
    _authoritative_floor_spend) caches the pre-reset DB spend for
    SPEND_DB_FLOOR_CACHE_TTL_SECONDS; left stale after a real reset, a request
    landing on the pod that cached it can read that higher floor and raise the
    counter right back above the just-reset spend. It is overwritten here with
    the post-reset floor (not merely deleted) and _authoritative_floor_spend
    re-checks the marker after its DB read, so a floor read already in flight
    on this pod when the reset commits cannot clobber it with the pre-reset
    value. Both keys are broadcast as SETs carrying new_spend, not deletes:
    every subscriber (remote pods AND this pod's own, which receives its own
    message) writes the post-reset value, so the self-delivered message cannot
    erase the guard just written here.

    Raises HTTPException(503) if Redis still holds the stale pre-reset counter
    after both the SET and the fallback DELETE fail: budget checks read Redis
    first, so returning success would leave the old value authoritative for
    every worker despite the DB write having committed.
    """
    from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import (
        evict_and_broadcast,
        publish_auth_cache_invalidation,
    )

    if new_spend is not None:
        from litellm.proxy.proxy_server import SPEND_DB_FLOOR_CACHE_TTL_SECONDS, spend_counter_cache

        spend_counter_key: Final = f"spend:team_member:{user_id}:{team_id}"
        spend_db_floor_key: Final = f"spend_db_floor:{spend_counter_key}"

        spend_counter_cache.in_memory_cache.set_cache(key=spend_counter_key, value=new_spend, ttl=60)
        if spend_counter_cache.redis_cache is not None:
            try:
                await spend_counter_cache.redis_cache.async_set_cache(key=spend_counter_key, value=new_spend, ttl=60)
            except Exception as e:  # noqa: BLE001  # fall back to deleting the stale entry before giving up
                verbose_proxy_logger.warning(
                    "Failed to set spend counter %s in Redis after reset: %s; deleting it instead so the next "
                    "read reseeds from the DB rather than keeping the stale pre-reset value authoritative",
                    spend_counter_key,
                    e,
                )
                try:
                    await spend_counter_cache.redis_cache.async_delete_cache(key=spend_counter_key)
                except Exception:  # noqa: BLE001  # stale value now authoritative in Redis; surface instead of reporting success
                    verbose_proxy_logger.warning(
                        "Failed to delete stale spend counter %s in Redis after a failed reset write",
                        spend_counter_key,
                        exc_info=True,
                    )
                    raise HTTPException(
                        status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
                        detail={  # mutable-ok: HTTPException.detail takes a dict
                            "error": "Spend was reset in the database, but Redis is unreachable and still "
                            "holds the pre-reset counter. Retry once Redis is reachable."
                        },
                    ) from e

        spend_counter_cache.in_memory_cache.set_cache(
            key=spend_db_floor_key,
            value=new_spend,
            ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS,
        )
        await publish_auth_cache_invalidation(cache_key=spend_counter_key, new_value=new_spend, ttl=60)
        await publish_auth_cache_invalidation(
            cache_key=spend_db_floor_key,
            new_value=new_spend,
            ttl=SPEND_DB_FLOOR_CACHE_TTL_SECONDS,
        )

    await evict_and_broadcast(
        cache_keys=(
            team_membership_auth_cache_key(team_id=team_id, user_id=user_id),
            team_membership_reservation_cache_key(user_id=user_id, team_id=team_id),
        ),
        user_api_key_cache=user_api_key_cache,
    )


async def delete_cache_team_object(
    team_id: str,
    team_alias: str | None,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None,
) -> None:
    """
    Evict both keys `_cache_team_object` writes.

    `get_team_object` reads the id key and the JWT `team_alias_jwt_field` path reads the alias key,
    so leaving either behind keeps a deleted team resolvable for auth until its TTL expires.

    Mirrors `delete_cached_project_object`: evicting locally only reaches the worker handling the
    delete, so every key is also broadcast to drop the other workers' in-memory copies.

    Eviction is best-effort, matching `_cache_team_object`. `delete_team` calls this after the team
    rows are already gone, so letting an unreachable cache backend raise here would fail a request
    whose delete has committed.
    """
    keys: Final = (f"team_id:{team_id}", *((f"team_alias:{team_alias}",) if team_alias else ()))

    for key in keys:
        try:
            user_api_key_cache.delete_cache(key=key)

            ## UPDATE REDIS CACHE ##
            if proxy_logging_obj is not None:
                await proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache(key=key)
        except Exception as e:  # noqa: BLE001  # best-effort invalidation: any cache backend error must not abort the delete
            verbose_proxy_logger.warning(
                "Failed to invalidate cached team entry %s on delete; "
                "a deleted team may be served until its TTL expires: %s",
                key,
                e,
            )
        await publish_auth_cache_invalidation(cache_key=key)


async def _cache_key_object(
    hashed_token: str,
    user_api_key_obj: UserAPIKeyAuth,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None,
):
    key: Final = hashed_token

    ## CACHE REFRESH TIME
    user_api_key_obj.last_refreshed_at = time.time()

    cached_key_obj: Final = _copy_user_api_key_auth_for_cache(user_api_key_obj=user_api_key_obj)
    await _cache_management_object(
        key=key,
        value=cached_key_obj,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
        model_type=UserAPIKeyAuth,
    )


async def _delete_cache_key_object(
    hashed_token: str,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None,
):
    key: Final = hashed_token

    user_api_key_cache.delete_cache(key=key)

    ## UPDATE REDIS CACHE ##
    if proxy_logging_obj is not None:
        await proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache(key=key)


async def delete_cache_key_objects(
    hashed_tokens: Sequence[str],
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None,
) -> None:
    """
    Evict a batch of key objects, for callers that delete keys in bulk rather than through
    `/key/delete`. Auth resolves a cached key object without re-reading its team, so a key left
    cached after its row is gone keeps buying access until its TTL expires.

    Evicting locally only reaches this worker, so each token is also broadcast: a deleted key left
    in a peer worker's in-memory cache still authenticates there until its TTL expires.

    Best-effort per key: the rows are already deleted by the time this runs, so an unreachable
    cache backend must not abort the caller partway through its own cascade.
    """
    results: Final = await asyncio.gather(
        *(
            _delete_cache_key_object(
                hashed_token=hashed_token,
                user_api_key_cache=user_api_key_cache,
                proxy_logging_obj=proxy_logging_obj,
            )
            for hashed_token in hashed_tokens
        ),
        return_exceptions=True,
    )

    for hashed_token, result in zip(hashed_tokens, results):
        if isinstance(result, BaseException):
            verbose_proxy_logger.warning(
                "Failed to evict cached key entry for %s; a deleted key may authenticate until its TTL expires: %s",
                hashed_token,
                result,
            )
        await publish_auth_cache_invalidation(cache_key=hashed_token)


class _TeamNotFoundDetail(TypedDict):
    error: ReadOnly[str]


class TeamNotFoundError(HTTPException):
    """The team row is provably absent, as opposed to merely unreadable.

    ``get_team_object`` reports every failure as a 404, so a deleted team and a
    database that would not answer are indistinguishable to its callers. Callers
    that must not treat a degraded read as a definitive answer, such as the
    authorization fallback in ``user_api_key_auth``, key on this subclass. It
    stays a 404 carrying the same detail, so every other caller is unaffected.
    """

    def __init__(self, team_id: str) -> None:
        detail: Final[_TeamNotFoundDetail] = {
            "error": f"Team doesn't exist in db. Team={team_id}. Create team via `/team/new` call."
        }
        super().__init__(status_code=404, detail=detail)


@log_db_metrics
async def _get_team_db_check(
    team_id: str, prisma_client: PrismaClient, team_id_upsert: bool | None = None
) -> "_PrismaTeamRow | None":
    response = await _team_table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id})

    if response is None and team_id_upsert:
        from litellm.proxy.management_endpoints.team_endpoints import new_team

        new_team_data: Final = NewTeamRequest(team_id=team_id)

        mock_request: Final = Request(scope={"type": "http"})
        system_admin_user: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN)

        created_team_dict: Final = await new_team(
            data=new_team_data,
            http_request=mock_request,
            user_api_key_dict=system_admin_user,
        )
        response = LiteLLM_TeamTable.model_validate(created_team_dict)
    return response


async def _get_team_object_from_db(team_id: str, prisma_client: PrismaClient) -> "_PrismaTeamRow | None":
    return await _team_table(TeamRepository(prisma_client)).find_unique(where={"team_id": team_id})


async def _get_team_object_from_user_api_key_cache(
    team_id: str,
    prisma_client: PrismaClient,
    user_api_key_cache: UserApiKeyCache,
    last_db_access_time: LimitedSizeOrderedDict,
    db_cache_expiry: int,
    proxy_logging_obj: ProxyLogging | None,
    key: str,
    team_id_upsert: bool | None = None,
) -> LiteLLM_TeamTableCachedObj:
    db_access_time_key: Final = key
    should_check_db: Final = _should_check_db(
        key=db_access_time_key,
        last_db_access_time=last_db_access_time,
        db_cache_expiry=db_cache_expiry,
    )
    if should_check_db:
        response = await _get_team_db_check(team_id=team_id, prisma_client=prisma_client, team_id_upsert=team_id_upsert)
        # The database answered and the row is not there. Distinct from every
        # other failure here, which leaves the team's grant unknown.
        if response is None:
            raise TeamNotFoundError(team_id=team_id)
    else:
        response = None

    if response is None:
        raise Exception

    _response: Final = LiteLLM_TeamTableCachedObj.model_validate(response.dict())

    # Load object_permission if object_permission_id exists but object_permission is not loaded
    if _response.object_permission_id and not _response.object_permission:
        try:
            _response.object_permission = await get_object_permission(
                object_permission_id=_response.object_permission_id,
                prisma_client=prisma_client,
                user_api_key_cache=user_api_key_cache,
                parent_otel_span=None,
                proxy_logging_obj=proxy_logging_obj,
            )
        except Exception as e:
            verbose_proxy_logger.debug(
                "Failed to load object_permission for team %s with object_permission_id=%s: %s",
                team_id,
                _response.object_permission_id,
                e,
            )

    # save the team object to cache
    await _cache_team_object(
        team_id=team_id,
        team_table=_response,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
    )

    # save to db access time
    _update_last_db_access_time(
        key=db_access_time_key,
        value=_response,
        last_db_access_time=last_db_access_time,
    )

    return _response


async def _get_team_object_from_cache(
    key: str,
    proxy_logging_obj: ProxyLogging | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None,
) -> LiteLLM_TeamTableCachedObj | None:
    ## INTERNAL USAGE CACHE (plain DualCache) — checked before UserApiKeyCache stores ##
    if proxy_logging_obj is not None and proxy_logging_obj.internal_usage_cache.dual_cache:
        cached_raw: Final = await proxy_logging_obj.internal_usage_cache.dual_cache.async_get_cache(
            key=key, parent_otel_span=parent_otel_span
        )
        if cached_raw is not None:
            from_internal: Final = CacheCodec.deserialize(cached_raw, LiteLLM_TeamTableCachedObj)
            if from_internal is not None:
                return from_internal

    decoded: Final = await user_api_key_cache.async_get_cache(
        key=key,
        parent_otel_span=parent_otel_span,
        model_type=LiteLLM_TeamTableCachedObj,
    )
    return decoded


async def get_team_object(
    team_id: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
    check_cache_only: bool | None = None,
    check_db_only: bool | None = None,
    team_id_upsert: bool | None = None,
) -> LiteLLM_TeamTableCachedObj:
    """
    - Check if team id in proxy Team Table
    - if valid, return LiteLLM_TeamTable object with defined limits
    - if not, then raise an error

    Raises:
        - HTTPException: If team doesn't exist in db or cache (status_code=404)
    """
    if prisma_client is None:
        raise Exception("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys")

    # check if in cache
    key: Final = f"team_id:{team_id}"

    if not check_db_only:
        cached_team_obj: Final = await _get_team_object_from_cache(
            key=key,
            proxy_logging_obj=proxy_logging_obj,
            user_api_key_cache=user_api_key_cache,
            parent_otel_span=parent_otel_span,
        )

        if cached_team_obj is not None:
            return cached_team_obj

        if check_cache_only:
            raise HTTPException(
                status_code=404,
                detail={"error": f"Team doesn't exist in cache + check_cache_only=True. Team={team_id}."},
            )

    # else, check db
    try:
        return await _get_team_object_from_user_api_key_cache(
            team_id=team_id,
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            proxy_logging_obj=proxy_logging_obj,
            last_db_access_time=last_db_access_time,
            db_cache_expiry=db_cache_expiry,
            key=key,
            team_id_upsert=team_id_upsert,
        )
    except TeamNotFoundError:
        raise
    except Exception:
        raise HTTPException(
            status_code=404,
            detail={"error": f"Team doesn't exist in db. Team={team_id}. Create team via `/team/new` call."},
        )


async def _cache_access_object(
    access_group_id: str,
    access_group_table: LiteLLM_AccessGroupTable,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None = None,
):
    key: Final = f"access_group_id:{access_group_id}"
    await user_api_key_cache.async_set_cache(
        key=key,
        value=access_group_table,
        model_type=LiteLLM_AccessGroupTable,
        ttl=DEFAULT_ACCESS_GROUP_CACHE_TTL,
    )


async def _delete_cache_access_object(
    access_group_id: str,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None = None,
):
    key: Final = f"access_group_id:{access_group_id}"

    user_api_key_cache.delete_cache(key=key)

    ## UPDATE REDIS CACHE ##
    if proxy_logging_obj is not None:
        await proxy_logging_obj.internal_usage_cache.dual_cache.async_delete_cache(key=key)


@log_db_metrics
async def get_access_object(
    access_group_id: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None = None,
) -> LiteLLM_AccessGroupTable:
    """
    - Check if access_group_id in proxy AccessGroupTable
    - Always checks cache first, then DB only when not found in cache
    - if valid, return LiteLLM_AccessGroupTable object
    - if not, then raise an error

    Unlike get_team_object, this has no check_cache_only or check_db_only flags;
    it always follows cache-first-then-db semantics.

    Raises:
        - HTTPException: If access group doesn't exist in db or cache (status_code=404)
    """
    if prisma_client is None:
        raise Exception("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys")

    key: Final = f"access_group_id:{access_group_id}"

    cached_access_obj: Final = await user_api_key_cache.async_get_cache(
        key=key,
        model_type=LiteLLM_AccessGroupTable,
    )
    if cached_access_obj is not None:
        return cached_access_obj

    # Not in cache - fetch from DB
    try:
        response: Final = await _dictable_table(AccessGroupRepository(prisma_client)).find_unique(
            where={"access_group_id": access_group_id}
        )

        if response is None:
            raise HTTPException(
                status_code=404,
                detail={"error": f"Access group doesn't exist in db. Access group={access_group_id}."},
            )

        _response: Final = LiteLLM_AccessGroupTable.model_validate(response.dict())

        # Save to cache
        await _cache_access_object(
            access_group_id=access_group_id,
            access_group_table=_response,
            user_api_key_cache=user_api_key_cache,
            proxy_logging_obj=proxy_logging_obj,
        )

        return _response
    except HTTPException:
        raise
    except Exception as e:
        verbose_proxy_logger.exception(
            "Error getting access group for access_group_id: %s",
            access_group_id,
        )
        raise HTTPException(
            status_code=404,
            detail={"error": f"Access group doesn't exist in db. Access group={access_group_id}. Error: {e}"},
        )


@log_db_metrics
async def get_team_object_by_alias(
    team_alias: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Optional["Span"] = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> LiteLLM_TeamTableCachedObj:
    """
    Look up a team by its team_alias (name) in the database.

    Args:
        team_alias: The team name/alias to look up
        prisma_client: Database client
        user_api_key_cache: Cache for storing results
        parent_otel_span: Optional OpenTelemetry span
        proxy_logging_obj: Optional proxy logging object

    Returns:
        LiteLLM_TeamTableCachedObj: The team object if found

    Raises:
        HTTPException: If team doesn't exist or multiple teams have the same alias
    """
    if prisma_client is None:
        raise Exception("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys")

    # Check cache first (keyed by alias)
    cache_key: Final = f"team_alias:{team_alias}"

    cached_team_obj: Final = await _get_team_object_from_cache(
        key=cache_key,
        proxy_logging_obj=proxy_logging_obj,
        user_api_key_cache=user_api_key_cache,
        parent_otel_span=parent_otel_span,
    )

    if cached_team_obj is not None:
        return cached_team_obj

    # Query database by team_alias
    try:
        teams: Final = await _team_table(TeamRepository(prisma_client)).find_many(where={"team_alias": team_alias})

        if not teams:
            raise HTTPException(
                status_code=404,
                detail={
                    "error": f"Team with alias '{team_alias}' doesn't exist in db. Create team via `/team/new` call."
                },
            )

        if len(teams) > 1:
            raise HTTPException(
                status_code=400,
                detail={
                    "error": f"Multiple teams found with alias '{team_alias}'. Please use team_id_jwt_field instead or ensure team aliases are unique."
                },
            )

        team: Final = teams[0]
        team_obj: Final = LiteLLM_TeamTableCachedObj.model_validate(team.model_dump())

        # Load object_permission if object_permission_id exists but object_permission is not loaded
        if team_obj.object_permission_id and not team_obj.object_permission:
            try:
                team_obj.object_permission = await get_object_permission(
                    object_permission_id=team_obj.object_permission_id,
                    prisma_client=prisma_client,
                    user_api_key_cache=user_api_key_cache,
                    parent_otel_span=parent_otel_span,
                    proxy_logging_obj=proxy_logging_obj,
                )
            except Exception as e:
                verbose_proxy_logger.debug(
                    "Failed to load object_permission for team %s with object_permission_id=%s: %s",
                    team_obj.team_id,
                    team_obj.object_permission_id,
                    e,
                )

        # Cache the result by both alias and team_id
        await user_api_key_cache.async_set_cache(
            key=cache_key,
            value=team_obj,
            model_type=LiteLLM_TeamTableCachedObj,
            ttl=DEFAULT_IN_MEMORY_TTL,
        )
        # Also cache by team_id for consistency
        team_id_cache_key: Final = f"team_id:{team_obj.team_id}"
        await user_api_key_cache.async_set_cache(
            key=team_id_cache_key,
            value=team_obj,
            model_type=LiteLLM_TeamTableCachedObj,
            ttl=DEFAULT_IN_MEMORY_TTL,
        )

        return team_obj

    except HTTPException:
        raise
    except Exception as e:
        verbose_proxy_logger.exception("Error looking up team by alias: %s", team_alias)
        raise HTTPException(
            status_code=500,
            detail={"error": f"Error looking up team by alias '{team_alias}': {e}"},
        )


@log_db_metrics
async def get_org_object_by_alias(
    org_alias: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Optional["Span"] = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> LiteLLM_OrganizationTable | None:
    """
    Look up an organization by its organization_alias in the database.

    Args:
        org_alias: The organization name/alias to look up
        prisma_client: Database client
        user_api_key_cache: Cache for storing results
        parent_otel_span: Optional OpenTelemetry span
        proxy_logging_obj: Optional proxy logging object

    Returns:
        LiteLLM_OrganizationTable if found, None otherwise

    Raises:
        HTTPException: If organization not found or multiple orgs have the same alias
    """
    if prisma_client is None:
        raise Exception("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys")

    # Check cache first (keyed by alias)
    cache_key: Final = f"org_alias:{org_alias}"
    cached_org_obj: Final = await user_api_key_cache.async_get_cache(
        key=cache_key,
        model_type=LiteLLM_OrganizationTable,
    )
    if cached_org_obj is not None:
        return cached_org_obj

    # Query database by organization_alias
    try:
        orgs = await _model_dump_table(OrganizationRepository(prisma_client)).find_many(
            where={"organization_alias": org_alias}
        )

        if not orgs:
            raise HTTPException(
                status_code=404,
                detail={
                    "error": f"Organization with alias '{org_alias}' doesn't exist in db. Create organization via `/organization/new` call."
                },
            )

        if len(orgs) > 1:
            raise HTTPException(
                status_code=400,
                detail={
                    "error": f"Multiple organizations found with alias '{org_alias}'. Please use org_id_jwt_field instead or ensure organization aliases are unique."
                },
            )

        org: Final = orgs[0]
        org_obj: Final = LiteLLM_OrganizationTable.model_validate(org.model_dump())

        # Cache the result
        await user_api_key_cache.async_set_cache(
            key=cache_key,
            value=org_obj,
            model_type=LiteLLM_OrganizationTable,
            ttl=DEFAULT_IN_MEMORY_TTL,
        )
        # Also cache by org_id for consistency
        await user_api_key_cache.async_set_cache(
            key=f"org_id:{org_obj.organization_id}",
            value=org_obj,
            model_type=LiteLLM_OrganizationTable,
            ttl=DEFAULT_IN_MEMORY_TTL,
        )

        return org_obj

    except HTTPException:
        raise
    except Exception as e:
        verbose_proxy_logger.exception("Error looking up organization by alias: %s", org_alias)
        raise HTTPException(
            status_code=500,
            detail={"error": f"Error looking up organization by alias '{org_alias}': {e}"},
        )


class ExperimentalUIJWTToken:
    @staticmethod
    def get_experimental_ui_login_jwt_auth_token(user_info: LiteLLM_UserTable) -> str:
        from datetime import timedelta

        from litellm.proxy.common_utils.encrypt_decrypt_utils import (
            encrypt_value_helper,
        )

        if user_info.user_role is None:
            raise Exception("User role is required for experimental UI login")

        # Experimental UI flow uses fixed 10-min expiry for security (does not use LITELLM_UI_SESSION_DURATION)
        expiration_time: Final = get_utc_datetime() + timedelta(minutes=10)

        # Format the expiration time as ISO 8601 string
        expires: Final = expiration_time.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "+00:00"

        valid_token: Final = UserAPIKeyAuth(
            token="ui-token",
            key_name="ui-token",
            key_alias="ui-token",
            max_budget=litellm.max_ui_session_budget,
            rpm_limit=100,  # allow user to have a conversation on test key pane of UI
            expires=expires,
            user_id=user_info.user_id,
            team_id="litellm-dashboard",
            models=user_info.models,
            max_parallel_requests=None,
            user_role=LitellmUserRoles(user_info.user_role),
        )

        return encrypt_value_helper(valid_token.model_dump_json(exclude_none=True))

    @staticmethod
    def get_cli_jwt_auth_token(
        user_info: LiteLLM_UserTable,
        team_id: str | None = None,
        team_alias: str | None = None,
        team_models: Sequence[str] | None = None,
        team_model_aliases: Mapping[str, str] | None = None,
        max_budget: float | None = None,
    ) -> str:
        """
        Generate a JWT token for CLI authentication with configurable expiration.

        The expiration time can be controlled via the LITELLM_CLI_JWT_EXPIRATION_HOURS
        environment variable (defaults to 24 hours).

        Args:
            user_info: User information from the database
            team_id: Team ID for the user (optional, uses user's team if available)
            team_alias: Team alias for the selected team, if available
            team_models: Model allowlist granted by the selected team
            team_model_aliases: Team model aliases for the selected team

        Returns:
            Encrypted JWT token string
        """
        import secrets
        from datetime import timedelta

        from litellm.proxy.common_utils.encrypt_decrypt_utils import (
            encrypt_value_helper,
        )

        if user_info.user_role is None:
            raise Exception("User role is required for CLI JWT login")

        # Calculate expiration time (configurable via LITELLM_CLI_JWT_EXPIRATION_HOURS env var)
        expiration_time: Final = get_utc_datetime() + timedelta(hours=CLI_JWT_EXPIRATION_HOURS)

        # Format the expiration time as ISO 8601 string
        expires: Final = expiration_time.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "+00:00"

        # Use provided team_id, or fall back to user's teams if available
        _team_id = team_id
        if _team_id is None and hasattr(user_info, "teams") and user_info.teams:
            # Use first team if user has teams
            _team_id = user_info.teams[0] if len(user_info.teams) > 0 else None

        session_token: Final = f"{CLI_SESSION_KEY_PREFIX}-{secrets.token_urlsafe(16)}"
        session_alias: Final = f"{CLI_SESSION_KEY_PREFIX}-{user_info.user_id}"

        valid_token: Final = UserAPIKeyAuth(
            token=session_token,
            key_name=session_alias,
            key_alias=session_alias,
            expires=expires,
            max_budget=max_budget,
            user_id=user_info.user_id,
            team_id=_team_id,
            team_alias=team_alias,
            team_models=list(team_models) if team_models is not None else [],
            team_model_aliases=dict(team_model_aliases) if team_model_aliases is not None else None,
            models=[] if _team_id is not None else user_info.models,
            max_parallel_requests=None,
            user_role=LitellmUserRoles(user_info.user_role),
            is_session_token=True,
        )

        return encrypt_value_helper(valid_token.model_dump_json(exclude_none=True))

    @staticmethod
    def get_key_object_from_ui_hash_key(
        hashed_token: str,
    ) -> UserAPIKeyAuth | None:
        import json

        from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth
        from litellm.proxy.common_utils.encrypt_decrypt_utils import (
            decrypt_value_helper,
        )

        decrypted_token: Final = decrypt_value_helper(hashed_token, key="ui_hash_key", exception_type="debug")
        if decrypted_token is None:
            return None
        try:
            return UserAPIKeyAuth.model_validate(json.loads(decrypted_token))
        except Exception as e:
            raise Exception(f"Invalid hash key. Hash key={hashed_token}. Decrypted token={decrypted_token}. Error: {e}")


async def _fetch_key_object_from_db_with_reconnect(
    hashed_token: str,
    prisma_client: PrismaClient,
    parent_otel_span: Span | None,
    proxy_logging_obj: ProxyLogging | None,
) -> BaseModel | None:
    """
    Fetch key object from DB and retry once if a DB connection error can be healed.
    """
    try:
        return await prisma_client.get_data(
            token=hashed_token,
            table_name="combined_view",
            parent_otel_span=parent_otel_span,
            proxy_logging_obj=proxy_logging_obj,
        )
    except Exception as e:
        if PrismaDBExceptionHandler.is_database_transport_error(e):
            did_reconnect = False
            if hasattr(prisma_client, "attempt_db_reconnect"):
                auth_reconnect_timeout = getattr(prisma_client, "_db_auth_reconnect_timeout_seconds", 2.0)
                if not isinstance(auth_reconnect_timeout, (int, float)):
                    auth_reconnect_timeout = 2.0
                auth_reconnect_lock_timeout = getattr(prisma_client, "_db_auth_reconnect_lock_timeout_seconds", 0.1)
                if not isinstance(auth_reconnect_lock_timeout, (int, float)):
                    auth_reconnect_lock_timeout = 0.1
                did_reconnect = await prisma_client.attempt_db_reconnect(
                    reason="auth_get_key_object_lookup_failure",
                    timeout_seconds=auth_reconnect_timeout,
                    lock_timeout_seconds=auth_reconnect_lock_timeout,
                )
            if did_reconnect:
                return await prisma_client.get_data(
                    token=hashed_token,
                    table_name="combined_view",
                    parent_otel_span=parent_otel_span,
                    proxy_logging_obj=proxy_logging_obj,
                )
        raise


@log_db_metrics
async def get_jwt_key_mapping_object(
    jwt_claim_name: str,
    jwt_claim_value: str,
    prisma_client: PrismaClient,
) -> str | None:
    """
    Lookup a JWT-to-virtual-key mapping from the database.

    Returns the hashed token (str) if a matching active mapping is found, else None.
    """
    mapping: Final = await _jwt_key_mapping_table(JWTKeyMappingRepository(prisma_client)).find_first(
        where={
            "jwt_claim_name": jwt_claim_name,
            "jwt_claim_value": jwt_claim_value,
            "is_active": True,
        }
    )
    if mapping is not None:
        return mapping.token
    return None


@log_db_metrics
async def get_key_object(
    hashed_token: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
    check_cache_only: bool | None = None,
) -> UserAPIKeyAuth:
    """
    - Check if team id in proxy Team Table
    - if valid, return LiteLLM_TeamTable object with defined limits
    - if not, then raise an error
    """
    if prisma_client is None:
        raise Exception("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys")

    # check if in cache
    key: Final = hashed_token

    # Same flow as before: use cache only when we have a hit we can turn into UserAPIKeyAuth
    # (dict from Redis / model_dump, or UserAPIKeyAuth from in-memory). Otherwise fall through to DB.
    user_api_key_auth: Final = await user_api_key_cache.async_get_cache(
        key=key,
        model_type=UserAPIKeyAuth,
    )
    if user_api_key_auth is not None:
        return _copy_user_api_key_auth_for_cache(user_api_key_obj=user_api_key_auth)

    if check_cache_only:
        raise Exception(f"Key doesn't exist in cache + check_cache_only=True. key={key}.")

    # else, check db
    _valid_token: Final[BaseModel | None] = await _fetch_key_object_from_db_with_reconnect(
        hashed_token=hashed_token,
        prisma_client=prisma_client,
        parent_otel_span=parent_otel_span,
        proxy_logging_obj=proxy_logging_obj,
    )

    if _valid_token is None:
        raise ProxyException(
            message=f"Authentication Error, Invalid proxy server token passed. key={hashed_token}, not found in db. Create key via `/key/generate` call.",
            type=ProxyErrorTypes.token_not_found_in_db,
            param="key",
            code=status.HTTP_401_UNAUTHORIZED,
        )

    _response: Final = UserAPIKeyAuth.model_validate(_valid_token.model_dump(exclude_none=True))

    # Load object_permission if object_permission_id exists but object_permission is not loaded
    if _response.object_permission_id and not _response.object_permission:
        try:
            _response.object_permission = await get_object_permission(
                object_permission_id=_response.object_permission_id,
                prisma_client=prisma_client,
                user_api_key_cache=user_api_key_cache,
                parent_otel_span=parent_otel_span,
                proxy_logging_obj=proxy_logging_obj,
            )
        except Exception as e:
            verbose_proxy_logger.debug(
                "Failed to load object_permission for key with object_permission_id=%s: %s",
                _response.object_permission_id,
                e,
            )

    # save the key object to cache
    await _cache_key_object(
        hashed_token=hashed_token,
        user_api_key_obj=_response,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
    )

    return _response


def _copy_user_api_key_auth_for_cache(
    user_api_key_obj: UserAPIKeyAuth,
) -> UserAPIKeyAuth:
    copied_key_obj: Final = user_api_key_obj.model_copy()
    copied_key_obj.budget_reservation = None
    copied_key_obj.budget_throttle_pct = None
    copied_key_obj.parent_otel_span = None
    copied_key_obj.request_route = None
    return copied_key_obj


@log_db_metrics
async def get_object_permission(
    object_permission_id: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> LiteLLM_ObjectPermissionTable | None:
    """
    - Check if object permission id in proxy ObjectPermissionTable
    - if valid, return LiteLLM_ObjectPermissionTable object
    - if not, then raise an error
    """
    if prisma_client is None:
        raise Exception("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys")

    # check if in cache
    key: Final = object_permission_cache_key(object_permission_id)
    deserialized_perm: Final = await user_api_key_cache.async_get_cache(
        key=key,
        model_type=LiteLLM_ObjectPermissionTable,
    )
    if deserialized_perm is not None:
        return deserialized_perm

    # else, check db
    try:
        response: Final = await _dictable_table(ObjectPermissionRepository(prisma_client)).find_unique(
            where={"object_permission_id": object_permission_id}
        )

        if response is None:
            return None

        _perm_obj: Final = LiteLLM_ObjectPermissionTable.model_validate(response.dict())
        await user_api_key_cache.async_set_cache(
            key=key,
            value=_perm_obj,
            model_type=LiteLLM_ObjectPermissionTable,
            ttl=get_management_object_ttl(user_api_key_cache),
        )

        return _perm_obj
    except Exception:
        return None


@log_db_metrics
async def get_managed_vector_store_rows_by_uuids(
    uuids: list[str],
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> list[LiteLLM_ManagedVectorStoresTable]:
    """
    Fetch managed vector store rows by their internal UUIDs.

    Follows the get_team_object / get_key_object / get_object_permission pattern:
    cache-first lookup (in-memory / Redis), DB fallback only on cache miss.
    Critical-path DB access must go through this helper to avoid raw Prisma
    calls on the hot request path.
    """
    if not uuids or prisma_client is None:
        return []

    result: Final[list[LiteLLM_ManagedVectorStoresTable]] = []
    cache_misses: Final[list[str]] = []

    for uuid in uuids:
        key = f"managed_vector_store_id:{uuid}"
        deserialized_vs = await user_api_key_cache.async_get_cache(
            key=key,
            model_type=LiteLLM_ManagedVectorStoresTable,
        )
        if deserialized_vs is not None:
            result.append(deserialized_vs)
        else:
            cache_misses.append(uuid)

    if not cache_misses:
        return result

    rows: Final = await _vector_store_table(ManagedVectorStoresRepository(prisma_client)).find_many(
        where={"vector_store_id": {"in": cache_misses}},
        take=len(cache_misses),
    )

    for row in rows:
        row_dict = row.model_dump() if hasattr(row, "model_dump") else (row.dict() if hasattr(row, "dict") else None)
        if not isinstance(row_dict, dict) or not row_dict:
            row_dict = dict(row) if hasattr(row, "__dict__") else {}
        if not row_dict:
            continue
        cached_obj = LiteLLM_ManagedVectorStoresTable.model_validate(row_dict)
        key = f"managed_vector_store_id:{cached_obj.vector_store_id}"
        await user_api_key_cache.async_set_cache(
            key=key,
            value=cached_obj,
            model_type=LiteLLM_ManagedVectorStoresTable,
            ttl=get_management_object_ttl(user_api_key_cache),
        )
        result.append(cached_obj)

    return result


class OrganizationNotFoundError(Exception):
    """The organization row is CONFIRMED absent, as opposed to a lookup that failed.

    Subclasses Exception so every existing except Exception caller keeps its current
    behavior; it exists so a caller that wants to treat "no such org" as "no restriction" can do
    that WITHOUT also swallowing an outage and silently dropping a real org ceiling.
    """


@log_db_metrics
async def get_org_object(
    org_id: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    parent_otel_span: Span | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
    include_budget_table: bool = False,
) -> LiteLLM_OrganizationTable | None:
    """
    - Check if org id in proxy Org Table
    - if valid, return LiteLLM_OrganizationTable object
    - if not, then raise an error

    Args:
        org_id: Organization ID to look up
        prisma_client: Database client
        user_api_key_cache: Cache for storing results
        parent_otel_span: Optional OpenTelemetry span
        proxy_logging_obj: Optional proxy logging object
        include_budget_table: If True, includes litellm_budget_table in the query
    """
    if prisma_client is None:
        raise Exception("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys")
    if not isinstance(org_id, str):
        return None

    # Use different cache key if budget table is included
    cache_key = f"org_id:{org_id}"
    if include_budget_table:
        cache_key = f"org_id:{org_id}:with_budget"

    # check if in cache
    deserialized_org: Final = await user_api_key_cache.async_get_cache(
        key=cache_key,
        model_type=LiteLLM_OrganizationTable,
    )
    if deserialized_org is not None:
        return deserialized_org
    # else, check db
    try:
        query_kwargs: Final[dict[str, Mapping[str, object]]] = {"where": {"organization_id": org_id}}
        if include_budget_table:
            query_kwargs["include"] = {"litellm_budget_table": True}

        response: Final = await _model_dump_table(OrganizationRepository(prisma_client)).find_unique(**query_kwargs)
    except Exception:
        # An operational failure (DB down, timeout, cache fault) is NOT the same fact as a confirmed
        # missing row, and relabelling it as "doesn't exist" made every caller unable to tell them
        # apart — a caller that treats absence as "this org places no restriction" then drops a real
        # org ceiling during an outage. Propagate the real error; callers that already catch
        # Exception are unaffected.
        raise

    if response is None:
        raise OrganizationNotFoundError(
            f"Organization doesn't exist in db. Organization={org_id}. Create organization via `/organization/new` call."
        )

    _org_obj: Final = LiteLLM_OrganizationTable.model_validate(response.model_dump())
    # Cache the result
    await user_api_key_cache.async_set_cache(
        key=cache_key,
        value=_org_obj,
        model_type=LiteLLM_OrganizationTable,
        ttl=DEFAULT_IN_MEMORY_TTL,
    )

    return _org_obj


async def _get_resources_from_access_groups(
    access_group_ids: Sequence[str],
    resource_field: Literal["access_model_names", "access_mcp_server_ids", "access_agent_ids"],
    prisma_client: PrismaClient | None = None,
    user_api_key_cache: UserApiKeyCache | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> list[str]:
    """
    Fetch access groups by their IDs (from cache or DB) and collect
    the specified resource field across all of them.

    Args:
        access_group_ids: List of access group IDs to fetch
        resource_field: Which resource list to extract from each access group
            - "access_model_names": model names (for model access checks)
            - "access_mcp_server_ids": MCP server IDs (for MCP access checks)
            - "access_agent_ids": agent IDs (for agent access checks)
        prisma_client: Optional PrismaClient (lazy-imported from proxy_server if None)
        user_api_key_cache: Optional DualCache (lazy-imported from proxy_server if None)
        proxy_logging_obj: Optional ProxyLogging (lazy-imported from proxy_server if None)

    Returns:
        Deduplicated list of resource identifiers from all resolved access groups.
    """
    if not access_group_ids:
        return []

    # Lazy import to avoid circular imports
    if prisma_client is None or user_api_key_cache is None:
        from litellm.proxy.proxy_server import prisma_client as _prisma_client
        from litellm.proxy.proxy_server import proxy_logging_obj as _proxy_logging_obj
        from litellm.proxy.proxy_server import user_api_key_cache as _user_api_key_cache

        prisma_client = prisma_client or _prisma_client
        user_api_key_cache = user_api_key_cache or _user_api_key_cache
        proxy_logging_obj = proxy_logging_obj or _proxy_logging_obj

    if user_api_key_cache is None:
        return []

    resources: Final[list[str]] = []
    for ag_id in access_group_ids:
        try:
            ag = await get_access_object(
                access_group_id=ag_id,
                prisma_client=prisma_client,
                user_api_key_cache=user_api_key_cache,
                proxy_logging_obj=proxy_logging_obj,
            )
            resources.extend(getattr(ag, resource_field, []))
        except Exception:
            verbose_proxy_logger.debug(
                "Could not fetch access group %s for resource field %s",
                ag_id,
                resource_field,
            )
    return list(set(resources))


async def _get_models_from_access_groups(
    access_group_ids: Sequence[str],
    prisma_client: PrismaClient | None = None,
    user_api_key_cache: UserApiKeyCache | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> list[str]:
    """
    Collect model names from unified access groups.
    Models are matched by model name for backwards compatibility.
    """
    return await _get_resources_from_access_groups(
        access_group_ids=access_group_ids,
        resource_field="access_model_names",
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
    )


async def _get_mcp_server_ids_from_access_groups(
    access_group_ids: list[str],
    prisma_client: PrismaClient | None = None,
    user_api_key_cache: UserApiKeyCache | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> list[str]:
    """
    Collect MCP server IDs from unified access groups.
    MCPs are matched by server ID.
    """
    return await _get_resources_from_access_groups(
        access_group_ids=access_group_ids,
        resource_field="access_mcp_server_ids",
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
    )


async def _get_agent_ids_from_access_groups(
    access_group_ids: list[str],
    prisma_client: PrismaClient | None = None,
    user_api_key_cache: UserApiKeyCache | None = None,
    proxy_logging_obj: ProxyLogging | None = None,
) -> list[str]:
    """
    Collect agent IDs from unified access groups.
    Agents are matched by agent ID.
    """
    return await _get_resources_from_access_groups(
        access_group_ids=access_group_ids,
        resource_field="access_agent_ids",
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
    )


def _resolve_all_team_model_sentinel_for_auth_check(
    models: list[str],
    llm_router: Router | None,
    team_id: str | None,
) -> list[str]:
    if SpecialModelNames.all_team_models.value not in models or team_id is None or llm_router is None:
        return models
    proxy_models: Final = llm_router.get_model_names()
    non_sentinel_models: Final = [model for model in models if model != SpecialModelNames.all_team_models.value]
    if not proxy_models:
        return non_sentinel_models or models
    return list(dict.fromkeys(non_sentinel_models + proxy_models))


def _check_model_access_helper(
    model: str,
    llm_router: Router | None,
    models: list[str],
    team_model_aliases: dict[str, str] | None = None,
    team_id: str | None = None,
) -> bool:
    ## check if model in allowed model names
    from collections import defaultdict

    access_groups: dict[str, list[str]] = defaultdict(list)

    if llm_router:
        access_groups = llm_router.get_model_access_groups(model_name=model, team_id=team_id)

    models = _resolve_all_team_model_sentinel_for_auth_check(
        models=models,
        llm_router=llm_router,
        team_id=team_id,
    )

    if len(access_groups) > 0 and llm_router is not None:  # check if token contains any model access groups
        for idx, m in enumerate(models):  # loop token models, if any of them are an access group add the access group
            if m in access_groups:
                return True

    # Filter out models that are access_groups
    filtered_models: Final = [m for m in models if m not in access_groups]

    if _model_in_team_aliases(model=model, team_model_aliases=team_model_aliases):
        return True

    if _model_matches_any_wildcard_pattern_in_list(model=model, allowed_model_list=filtered_models):
        return True

    all_model_access: bool = False

    if (len(filtered_models) == 0 and len(models) == 0) or "*" in filtered_models:
        all_model_access = True

    if SpecialModelNames.all_proxy_models.value in filtered_models:
        all_model_access = True

    if model is not None and model not in filtered_models and all_model_access is False:
        return False
    return True


def _can_object_call_model(
    model: str | list[str],
    llm_router: Router | None,
    models: list[str],
    team_model_aliases: dict[str, str] | None = None,
    team_id: str | None = None,
    object_type: Literal["user", "team", "key", "org", "project"] = "user",
    fallback_depth: int = 0,
) -> Literal[True]:
    """
    Checks if token can call a given model

    Args:
        - model: str
        - llm_router: Optional[Router]
        - models: List[str]
        - team_model_aliases: Optional[Dict[str, str]]
        - object_type: Literal["user", "team", "key", "org"]. We use the object type to raise the correct exception type

    Returns:
        - True: if token allowed to call model

    Raises:
        - Exception: If token not allowed to call model
    """
    if fallback_depth >= DEFAULT_MAX_RECURSE_DEPTH:
        raise Exception(f"Unable to parse model, max fallback depth exceeded - received model: {model}")
    if isinstance(model, list):
        for m in model:
            _can_object_call_model(
                model=m,
                llm_router=llm_router,
                models=models,
                team_model_aliases=team_model_aliases,
                team_id=team_id,
                object_type=object_type,
                fallback_depth=fallback_depth + 1,
            )
        return True

    potential_models: Final = [model]
    if model in litellm.model_alias_map:
        potential_models.append(litellm.model_alias_map[model])
    elif llm_router and model in llm_router.model_group_alias:
        _model: Final = llm_router._get_model_from_alias(model)
        if _model:
            potential_models.append(_model)

    ## check model access for alias + underlying model - allow if either is in allowed models
    for m in potential_models:
        if _check_model_access_helper(
            model=m,
            llm_router=llm_router,
            models=models,
            team_model_aliases=team_model_aliases,
            team_id=team_id,
        ):
            return True

    raise ProxyException(
        message=f"{object_type} not allowed to access model. This {object_type} can only access models={models}. Tried to access {model}",
        type=ProxyErrorTypes.get_model_access_error_type_for_object(object_type=object_type),
        param="model",
        code=status.HTTP_403_FORBIDDEN,
    )


def _model_in_team_aliases(model: str, team_model_aliases: dict[str, str] | None = None) -> bool:
    """
    Returns True if `model` being accessed is an alias of a team model

    - `model=gpt-4o`
    - `team_model_aliases={"gpt-4o": "gpt-4o-team-1"}`
        - returns True

    - `model=gp-4o`
    - `team_model_aliases={"o-3": "o3-preview"}`
        - returns False
    """
    if team_model_aliases:
        if model in team_model_aliases:
            return True
    return False


def _resolve_key_models_for_auth_check(valid_token: UserAPIKeyAuth) -> list[str]:
    """
    Expand key model sentinels before auth checks.

    ``all-team-models`` means inherit the parent team's allowlist -- same
    semantics as ``get_key_models`` in ``model_checks.py``.

    If the key has no team_id, it inherits the full proxy model list
    (equivalent to an empty models field, i.e. unrestricted access).
    """
    models: Final = list(valid_token.models or [])
    if SpecialModelNames.all_team_models.value in models:
        if valid_token.team_id is None:
            return []
        return list(valid_token.team_models or [])
    return models


async def can_key_call_model(
    model: str | list[str],
    llm_model_list: list | None,
    valid_token: UserAPIKeyAuth,
    llm_router: litellm.Router | None,
) -> Literal[True]:
    """
    Checks if token can call a given model

    1. First checks native key-level model permissions (current implementation)
    2. If not allowed natively, falls back to access_group_ids on the key

    Returns:
        - True: if token allowed to call model

    Raises:
        - Exception: If token not allowed to call model
    """
    key_models: Final = _resolve_key_models_for_auth_check(valid_token=valid_token)
    try:
        return _can_object_call_model(
            model=model,
            llm_router=llm_router,
            models=key_models,
            team_model_aliases=valid_token.team_model_aliases,
            team_id=valid_token.team_id,
            object_type="key",
        )
    except ProxyException:
        # Fallback: check key's access_group_ids
        key_access_group_ids: Final = valid_token.access_group_ids or []
        if key_access_group_ids:
            models_from_groups: Final = await _get_models_from_access_groups(
                access_group_ids=key_access_group_ids,
            )
            if models_from_groups:
                return _can_object_call_model(
                    model=model,
                    llm_router=llm_router,
                    models=models_from_groups,
                    team_model_aliases=valid_token.team_model_aliases,
                    team_id=valid_token.team_id,
                    object_type="key",
                )
        raise


async def can_key_call_resolved_model(
    model: str,
    llm_model_list: list | None,
    valid_token: UserAPIKeyAuth,
    llm_router: litellm.Router | None,
) -> None:
    from litellm.proxy.proxy_server import (
        prisma_client,
        proxy_logging_obj,
        user_api_key_cache,
    )

    skip_key_model_check: Final = valid_token.config or (
        isinstance(valid_token.models, list) and SpecialModelNames.all_team_models.value in valid_token.models
    )
    if not skip_key_model_check:
        await can_key_call_model(
            model=model,
            llm_model_list=llm_model_list,
            valid_token=valid_token,
            llm_router=llm_router,
        )

    team_object: LiteLLM_TeamTableCachedObj | None = None
    team_object_from_lookup = False
    if valid_token.team_id is not None:
        try:
            team_object = await get_team_object(
                team_id=valid_token.team_id,
                prisma_client=prisma_client,
                user_api_key_cache=user_api_key_cache,
                parent_otel_span=valid_token.parent_otel_span,
                proxy_logging_obj=proxy_logging_obj,
            )
            team_object_from_lookup = True
        except Exception:
            team_object = LiteLLM_TeamTableCachedObj(
                team_id=valid_token.team_id,
                models=valid_token.team_models,
                blocked=valid_token.team_blocked,
                team_alias=valid_token.team_alias,
                metadata=valid_token.team_metadata,
                object_permission_id=valid_token.team_object_permission_id,
                object_permission=valid_token.team_object_permission,
            )

    if team_object is not None:
        try:
            await can_team_access_model(
                model=model,
                team_object=team_object,
                llm_router=llm_router,
                team_model_aliases=valid_token.team_model_aliases,
            )
        except ProxyException as team_denial:
            if team_denial.type != ProxyErrorTypes.team_model_access_denied:
                raise
            if not await _key_access_group_grants_model(
                model=model,
                valid_token=valid_token,
                team_object=team_object,
                llm_router=llm_router,
            ):
                raise

        if valid_token.user_id is not None and team_object_from_lookup:
            await _check_team_member_model_access(
                model=model,
                team_object=team_object,
                valid_token=valid_token,
                llm_router=llm_router,
                prisma_client=prisma_client,
                user_api_key_cache=user_api_key_cache,
                proxy_logging_obj=proxy_logging_obj,
            )

    if valid_token.project_id is not None:
        project_object: Final = await get_project_object(
            project_id=valid_token.project_id,
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            proxy_logging_obj=proxy_logging_obj,
        )
        if project_object is not None and len(project_object.models) > 0:
            can_project_access_model(
                model=model,
                project_object=project_object,
                llm_router=llm_router,
            )


def can_org_access_model(
    model: str,
    org_object: LiteLLM_OrganizationTable | None,
    llm_router: Router | None,
    team_model_aliases: dict[str, str] | None = None,
) -> Literal[True]:
    """
    Returns True if the team can access a specific model.

    """
    return _can_object_call_model(
        model=model,
        llm_router=llm_router,
        models=org_object.models if org_object else [],
        team_model_aliases=team_model_aliases,
        object_type="org",
    )


async def can_team_access_model(
    model: str | list[str],
    team_object: LiteLLM_TeamTable | None,
    llm_router: Router | None,
    team_model_aliases: dict[str, str] | None = None,
) -> Literal[True]:
    """
    Returns True if the team can access a specific model.

    1. First checks native team-level model permissions (current implementation)
    2. If not allowed natively, falls back to access_group_ids on the team
    """
    try:
        return _can_object_call_model(
            model=model,
            llm_router=llm_router,
            models=team_object.models if team_object else [],
            team_model_aliases=team_model_aliases,
            team_id=team_object.team_id if team_object else None,
            object_type="team",
        )
    except ProxyException:
        # Fallback: check team's access_group_ids
        team_access_group_ids: Final = (team_object.access_group_ids or []) if team_object else []
        if team_access_group_ids:
            models_from_groups: Final = await _get_models_from_access_groups(
                access_group_ids=team_access_group_ids,
            )
            if models_from_groups:
                return _can_object_call_model(
                    model=model,
                    llm_router=llm_router,
                    models=models_from_groups,
                    team_model_aliases=team_model_aliases,
                    team_id=team_object.team_id if team_object else None,
                    object_type="team",
                )
        raise


async def get_authorized_resources_from_key_access_groups(
    valid_token: UserAPIKeyAuth | None,
    team_object: LiteLLM_TeamTable | None,
    resource_field: Literal["access_model_names", "access_mcp_server_ids", "access_agent_ids"],
) -> list[str]:
    """
    For each access_group_id on the key, fetch the LiteLLM_AccessGroupTable row
    and contribute its `resource_field` only if the group authorizes the caller
    as an owner — that is, the group's `assigned_team_ids` includes the key's
    `team_id`, or the group's `assigned_key_ids` includes the key's token. This
    preserves the team-as-owner boundary while still letting a group reach the
    key without first being added to the team's `access_group_ids` list.
    """
    if valid_token is None:
        return []
    key_access_group_ids: Final = list(valid_token.access_group_ids or [])
    if not key_access_group_ids:
        return []

    from litellm.proxy.proxy_server import prisma_client as _prisma_client
    from litellm.proxy.proxy_server import proxy_logging_obj as _proxy_logging_obj
    from litellm.proxy.proxy_server import user_api_key_cache as _user_api_key_cache

    if _prisma_client is None or _user_api_key_cache is None:
        return []

    key_team_id: Final = valid_token.team_id or (team_object.team_id if team_object is not None else None)
    key_token: Final = valid_token.token

    authorized_resources: Final[list[str]] = []
    for ag_id in key_access_group_ids:
        try:
            ag = await get_access_object(
                access_group_id=ag_id,
                prisma_client=_prisma_client,
                user_api_key_cache=_user_api_key_cache,
                proxy_logging_obj=_proxy_logging_obj,
            )
        except Exception:
            continue
        team_authorized = bool(key_team_id and key_team_id in (ag.assigned_team_ids or []))
        key_authorized = bool(key_token and key_token in (ag.assigned_key_ids or []))
        if team_authorized or key_authorized:
            authorized_resources.extend(getattr(ag, resource_field, []) or [])

    return list(set(authorized_resources))


async def _key_access_group_grants_model(
    model: str | list[str],
    valid_token: UserAPIKeyAuth | None,
    team_object: LiteLLM_TeamTable | None,
    llm_router: Router | None,
) -> bool:
    """
    Returns True if the key's `access_group_ids` expand to models that grant
    access to `model`. Used to let a key's access group override a team's
    model restriction in `common_checks`.
    """
    authorized_models: Final = await get_authorized_resources_from_key_access_groups(
        valid_token=valid_token,
        team_object=team_object,
        resource_field="access_model_names",
    )
    if not authorized_models:
        return False
    try:
        _can_object_call_model(
            model=model,
            llm_router=llm_router,
            models=authorized_models,
            team_model_aliases=valid_token.team_model_aliases if valid_token else None,
            team_id=valid_token.team_id if valid_token else None,
            object_type="key",
        )
        return True
    except ProxyException:
        return False


def can_project_access_model(
    model: str | list[str],
    project_object: LiteLLM_ProjectTableCachedObj,
    llm_router: Router | None,
) -> Literal[True]:
    """
    Returns True if the project can access a specific model.

    Raises ProxyException if access is denied.
    """
    return _can_object_call_model(
        model=model,
        llm_router=llm_router,
        models=project_object.models if project_object else [],
        object_type="project",
    )


async def can_user_call_model(
    model: str | list[str],
    llm_router: Router | None,
    user_object: LiteLLM_UserTable | None,
) -> Literal[True]:
    if user_object is None:
        return True

    if SpecialModelNames.no_default_models.value in user_object.models:
        raise ProxyException(
            message=f"User not allowed to access model. No default model access, only team models allowed. Tried to access {model}",
            type=ProxyErrorTypes.key_model_access_denied,
            param="model",
            code=status.HTTP_403_FORBIDDEN,
        )

    return _can_object_call_model(
        model=model,
        llm_router=llm_router,
        models=user_object.models,
        object_type="user",
    )


def _search_tool_names_from_object_permission(
    object_permission: LiteLLM_ObjectPermissionTable | None,
) -> list[str]:
    """Return allowlisted search tool names from object_permission (empty = unrestricted)."""
    if object_permission is None:
        return []
    raw: Final = object_permission.search_tools
    if not raw:
        return []
    return list(raw)


def _can_object_call_search_tool(
    search_tool_name: str,
    allowed_search_tools: list[str],
    object_type: Literal["key", "team", "project"],
) -> Literal[True]:
    """
    Check if an object (key/team/project) can access a specific search tool.

    Similar to _can_object_call_model but for search tools.

    Args:
        search_tool_name: The search tool being requested
        allowed_search_tools: List of allowed search tool names for this object
        object_type: Type of object for error messaging

    Returns:
        True if access is allowed

    Raises:
        ProxyException if access is denied
    """
    # Empty list means all search tools are allowed
    if not allowed_search_tools:
        return True

    # Check if the search tool is in the allowlist
    if search_tool_name in allowed_search_tools:
        return True

    # Access denied
    raise ProxyException(
        message=f"{object_type.capitalize()} not allowed to access search tool: {search_tool_name}. "
        f"Allowed search tools: {allowed_search_tools}",
        type=ProxyErrorTypes.key_model_access_denied,
        param="search_tool_name",
        code=status.HTTP_403_FORBIDDEN,
    )


async def can_key_call_search_tool(
    search_tool_name: str,
    valid_token: UserAPIKeyAuth,
) -> Literal[True]:
    """
    Check if a key can access a specific search tool.

    Similar to can_key_call_model but for search tools.

    Args:
        search_tool_name: The search tool being requested
        valid_token: The authenticated key

    Returns:
        True if access is allowed

    Raises:
        ProxyException if access is denied
    """
    return _can_object_call_search_tool(
        search_tool_name=search_tool_name,
        allowed_search_tools=_search_tool_names_from_object_permission(valid_token.object_permission),
        object_type="key",
    )


async def can_team_call_search_tool(
    search_tool_name: str,
    team_object: LiteLLM_TeamTable | None,
) -> Literal[True]:
    """
    Check if a team can access a specific search tool.

    Similar to can_team_access_model but for search tools.

    Args:
        search_tool_name: The search tool being requested
        team_object: The team object

    Returns:
        True if access is allowed

    Raises:
        ProxyException if access is denied
    """
    if team_object is None:
        return True

    return _can_object_call_search_tool(
        search_tool_name=search_tool_name,
        allowed_search_tools=_search_tool_names_from_object_permission(team_object.object_permission),
        object_type="team",
    )


async def can_user_view_search_tool(
    search_tool_name: str,
    valid_token: UserAPIKeyAuth,
    team_object: LiteLLM_TeamTable | None,
) -> bool:
    """
    Boolean variant of the key + team authorization enforced on /search, used to
    scope /search_tools/list so a non-admin caller only sees tools it may invoke.
    """
    try:
        await can_key_call_search_tool(
            search_tool_name=search_tool_name,
            valid_token=valid_token,
        )
        await can_team_call_search_tool(
            search_tool_name=search_tool_name,
            team_object=team_object,
        )
    except ProxyException:
        return False
    return True


async def is_valid_fallback_model(
    model: str,
    llm_router: Router | None,
    user_model: str | None,
) -> Literal[True]:
    """
    Try to route the fallback model request.

    Validate if it can't be routed.

    Help catch invalid fallback models.
    """
    await route_request(
        data={
            "model": model,
            "messages": [{"role": "user", "content": "Who was Alexander?"}],
        },
        llm_router=llm_router,
        user_model=user_model,
        route_type="acompletion",  # route type shouldn't affect the fallback model check
    )

    return True


def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool:
    """
    Throttle an over-budget key instead of blocking it, when the key opted in
    via `throttle_on_budget_exceeded` and a global percentage is configured.

    Records the percentage on the request-scoped `budget_throttle_pct` so the
    rate limiter scales the key's TPM/RPM down to it; the persistent limits are
    left untouched so the throttle never compounds across requests. Returns True
    when the key was throttled (caller skips raising), False when it should still
    be hard-blocked.
    """
    pct: Final = budget_throttle_percentage()
    if pct is None or not should_throttle_budget_exceeded(valid_token):
        return False
    valid_token.budget_throttle_pct = pct
    return True


async def _virtual_key_max_budget_check(
    valid_token: UserAPIKeyAuth,
    proxy_logging_obj: ProxyLogging,
    user_obj: LiteLLM_UserTable | None = None,
):
    """
    Raises:
        BudgetExceededError if the token is over it's max budget.
        Triggers a budget alert if the token is over it's max budget.

    """
    if valid_token.max_budget is not None:
        from litellm.proxy.proxy_server import get_current_spend

        fallback_spend: Final = valid_token.spend or 0.0
        counter_key: Final = f"spend:key:{valid_token.token}"

        # Read spend from cross-pod counter (Redis-first) or cached object (fallback)
        spend: Final = await get_current_spend(
            counter_key=counter_key,
            fallback_spend=fallback_spend,
            max_budget=valid_token.max_budget,
        )

        ####################################
        # collect information for alerting #
        ####################################

        user_email = None
        # Check if the token has any user id information
        if user_obj is not None:
            user_email = user_obj.user_email

        call_info: Final = CallInfo(
            token=valid_token.token,
            spend=spend,
            max_budget=valid_token.max_budget,
            soft_budget=valid_token.soft_budget,
            user_id=valid_token.user_id,
            team_id=valid_token.team_id,
            organization_id=valid_token.org_id,
            user_email=user_email,
            key_alias=valid_token.key_alias,
            event_group=Litellm_EntityType.KEY,
        )
        asyncio.create_task(
            proxy_logging_obj.budget_alerts(
                type="token_budget",
                user_info=call_info,
            )
        )

        ####################################
        # collect information for alerting #
        ####################################

        # Defense-in-depth (GHSA-2rv4-xv66-fpjg): spend >= NaN is always False,
        # so a NaN max_budget would silently disable enforcement.  Treat a
        # non-finite max_budget as "no configured limit" rather than as a bypass.
        if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget:
            if _apply_budget_exceeded_throttle(valid_token):
                return
            # name the key in the error so operators don't have to reverse-map
            # spend back to a key; key_name is the masked form (last 4 chars)
            key_label: Final = valid_token.key_alias or "key"
            key_descriptor: Final = f"{key_label} ({valid_token.key_name})" if valid_token.key_name else key_label
            raise litellm.BudgetExceededError(
                current_cost=spend,
                max_budget=valid_token.max_budget,
                message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}",
                entity_type=Litellm_EntityType.KEY.value,
                entity_id=valid_token.token,
            )


async def _virtual_key_multi_budget_check(
    valid_token: UserAPIKeyAuth,
):
    """
    Raises BudgetExceededError if any budget window in valid_token.budget_limits is exceeded.

    Each window has its own Redis counter keyed by spend:key:{token}:window:{budget_duration}.
    Using budget_duration (not list index) keeps counters stable when windows are reordered
    or removed during a key update.

    Note: counters are not seeded from DB on Redis cold-start. After a Redis flush,
    per-window spend resets to zero within the current window period. This is an acceptable
    trade-off: the DB stores reset_at timestamps but not per-window accumulated spend.
    """
    if not valid_token.budget_limits:
        return

    from litellm.proxy.proxy_server import get_current_spend

    for window in valid_token.budget_limits:
        w: dict = window if isinstance(window, dict) else window.model_dump()
        counter_key = f"spend:key:{valid_token.token}:window:{w['budget_duration']}"
        window_spend = await get_current_spend(
            counter_key=counter_key,
            fallback_spend=0.0,
            max_budget=w["max_budget"],
            window_entity_type="Key",
            window_entity_id=valid_token.token,
            window_start=get_budget_window_start(w),
        )
        if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]:
            raise litellm.BudgetExceededError(
                current_cost=window_spend,
                max_budget=w["max_budget"],
                message=(
                    f"ExceededBudget: Key over {w['budget_duration']} budget. "
                    f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
                ),
                entity_type=Litellm_EntityType.KEY.value,
                entity_id=valid_token.token,
            )


async def _virtual_key_soft_budget_check(
    valid_token: UserAPIKeyAuth,
    proxy_logging_obj: ProxyLogging,
    user_obj: LiteLLM_UserTable | None = None,
):
    """
    Triggers a budget alert if the token is over it's soft budget.

    """

    if valid_token.soft_budget and valid_token.spend >= valid_token.soft_budget:
        verbose_proxy_logger.debug(
            "Crossed Soft Budget for token %s, spend %s, soft_budget %s",
            valid_token.token,
            valid_token.spend,
            valid_token.soft_budget,
        )
        call_info: Final = CallInfo(
            token=valid_token.token,
            spend=valid_token.spend,
            max_budget=valid_token.max_budget,
            soft_budget=valid_token.soft_budget,
            user_id=valid_token.user_id,
            team_id=valid_token.team_id,
            team_alias=valid_token.team_alias,
            organization_id=valid_token.org_id,
            user_email=user_obj.user_email if user_obj else None,
            key_alias=valid_token.key_alias,
            event_group=Litellm_EntityType.KEY,
        )

        asyncio.create_task(
            proxy_logging_obj.budget_alerts(
                type="soft_budget",
                user_info=call_info,
            )
        )


def _parse_email_list(raw: str | Sequence[object] | None) -> list[str]:
    """Parse emails from a list or comma-separated string."""
    if isinstance(raw, list):
        return [e.strip() for e in raw if isinstance(e, str) and e.strip()]
    elif isinstance(raw, str):
        return [e.strip() for e in raw.split(",") if e.strip()]
    return []


def _normalize_alert_emails(
    cfg: Mapping[str, str | Sequence[object] | None] | None,
) -> dict[str, list[str]]:
    """Coerce user-supplied threshold→recipients mapping to Dict[str, List[str]].

    Values may legitimately arrive as list, comma-separated string, or None
    from YAML/metadata; _parse_email_list tolerates all three.
    """
    if not cfg:
        return {}
    return {k: _parse_email_list(v) for k, v in cfg.items()}


def _merge_budget_alert_email_configs(
    global_cfg: Mapping[str, str | Sequence[object] | None] | None,
    per_key_cfg: Mapping[str, str | Sequence[object] | None] | None,
) -> dict[str, list[str]] | None:
    """
    Per-threshold additive merge: each threshold's recipient list is the union
    of global + per-key entries (deduped, global-first ordering). Missing
    thresholds on one side are inherited from the other.
    """
    global_cfg_normalized: Final = _normalize_alert_emails(global_cfg)
    per_key_cfg_normalized: Final = _normalize_alert_emails(per_key_cfg)
    if not global_cfg_normalized and not per_key_cfg_normalized:
        return None
    thresholds: Final = set(global_cfg_normalized) | set(per_key_cfg_normalized)
    return {
        t: list(dict.fromkeys(global_cfg_normalized.get(t, []) + per_key_cfg_normalized.get(t, []))) for t in thresholds
    }


async def _virtual_key_max_budget_alert_check(
    valid_token: UserAPIKeyAuth,
    proxy_logging_obj: ProxyLogging,
    user_obj: LiteLLM_UserTable | None = None,
):
    """
    Triggers a budget alert if the token has reached EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE
    (default 80%) of its max budget.
    This is a warning alert before the token actually exceeds the max budget.

    """

    if valid_token.max_budget is not None and valid_token.spend is not None and valid_token.spend > 0:
        owner_email: Final = user_obj.user_email if user_obj else None
        alert_email_config: Final[dict[str, list[str]] | None] = _merge_budget_alert_email_configs(
            global_cfg=litellm.default_key_max_budget_alert_emails,
            per_key_cfg=(valid_token.metadata or {}).get("max_budget_alert_emails"),
        )

        if isinstance(alert_email_config, dict) and alert_email_config:
            # New path: only create task if spend has crossed the lowest threshold
            min_pct: Final = min(
                (int(k) for k in alert_email_config if k.isdigit()),
                default=None,
            )
            if min_pct is None or valid_token.spend < valid_token.max_budget * (min_pct / 100.0):
                return

            call_info = CallInfo(
                token=valid_token.token,
                spend=valid_token.spend,
                max_budget=valid_token.max_budget,
                soft_budget=valid_token.soft_budget,
                user_id=valid_token.user_id,
                team_id=valid_token.team_id,
                team_alias=valid_token.team_alias,
                organization_id=valid_token.org_id,
                user_email=owner_email,
                key_alias=valid_token.key_alias,
                event_group=Litellm_EntityType.KEY,
                max_budget_alert_emails=alert_email_config,
            )
            asyncio.create_task(
                proxy_logging_obj.budget_alerts(
                    type="max_budget_alert",
                    user_info=call_info,
                )
            )
        else:
            # Old path: existing single 80% threshold — completely unchanged
            alert_threshold: Final = valid_token.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE

            if valid_token.spend >= alert_threshold and valid_token.spend < valid_token.max_budget:
                verbose_proxy_logger.debug(
                    "Reached Max Budget Alert Threshold for token %s, spend %s, max_budget %s, alert_threshold %s",
                    valid_token.token,
                    valid_token.spend,
                    valid_token.max_budget,
                    alert_threshold,
                )
                call_info = CallInfo(
                    token=valid_token.token,
                    spend=valid_token.spend,
                    max_budget=valid_token.max_budget,
                    soft_budget=valid_token.soft_budget,
                    user_id=valid_token.user_id,
                    team_id=valid_token.team_id,
                    team_alias=valid_token.team_alias,
                    organization_id=valid_token.org_id,
                    user_email=owner_email,
                    key_alias=valid_token.key_alias,
                    event_group=Litellm_EntityType.KEY,
                )

                asyncio.create_task(
                    proxy_logging_obj.budget_alerts(
                        type="max_budget_alert",
                        user_info=call_info,
                    )
                )


async def _check_team_member_budget(
    team_object: LiteLLM_TeamTable | None,
    user_object: LiteLLM_UserTable | None,
    valid_token: UserAPIKeyAuth | None,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging,
):
    """Check if team member is over their max budget within the team."""
    if (
        team_object is not None
        and team_object.team_id is not None
        and valid_token is not None
        and valid_token.user_id is not None
    ):
        team_membership: Final = await get_team_membership(
            user_id=valid_token.user_id,
            team_id=team_object.team_id,
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            proxy_logging_obj=proxy_logging_obj,
        )

        # Per-member override wins; otherwise fall back to the team-level
        # default configured via team.metadata["team_member_budget_id"].
        team_member_budget: float | None = None
        if (
            team_membership is not None
            and team_membership.litellm_budget_table is not None
            and team_membership.litellm_budget_table.max_budget is not None
        ):
            team_member_budget = team_membership.litellm_budget_table.max_budget
        else:
            default_budget_id: Final = (team_object.metadata or {}).get("team_member_budget_id")
            if isinstance(default_budget_id, str):
                default_budget: Final = await get_team_member_default_budget(
                    budget_id=default_budget_id,
                    prisma_client=prisma_client,
                    user_api_key_cache=user_api_key_cache,
                )
                # Treat 0 on the team default as "no cap".
                # Per-member rows still respect 0 as an explicit admin disable.
                if (
                    default_budget is not None
                    and default_budget.max_budget is not None
                    and default_budget.max_budget > 0
                ):
                    team_member_budget = default_budget.max_budget

        if team_member_budget is not None:
            team_member_spend = (team_membership.spend if team_membership is not None else 0.0) or 0.0

            # Read from cross-pod counter (Redis-first) if available
            from litellm.proxy.proxy_server import get_current_spend

            team_member_spend = await get_current_spend(
                counter_key=f"spend:team_member:{valid_token.user_id}:{team_object.team_id}",
                fallback_spend=team_member_spend,
                max_budget=team_member_budget,
            )

            if math.isfinite(team_member_budget) and team_member_spend >= team_member_budget:
                raise litellm.BudgetExceededError(
                    current_cost=team_member_spend,
                    max_budget=team_member_budget,
                    message=f"Budget has been exceeded! User={valid_token.user_id} in Team={team_object.team_id} Current cost: {team_member_spend}, Max budget: {team_member_budget}",
                    entity_type=Litellm_EntityType.TEAM_MEMBER.value,
                    entity_id=f"{valid_token.user_id}:{team_object.team_id}",
                )


async def _check_team_member_model_access(
    model: str | list[str],
    team_object: LiteLLM_TeamTable,
    valid_token: UserAPIKeyAuth,
    llm_router: Router | None,
    prisma_client: Optional["PrismaClient"],
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging,
) -> None:
    """
    Check if a team member's per-member model scope allows access to the requested model.

    Only enforced when the member's budget table has a non-empty allowed_models list.
    If allowed_models is empty or absent, the team-level models list applies (no extra restriction).
    """
    if valid_token.user_id is None or team_object.team_id is None:
        return

    team_membership: Final = await get_team_membership(
        user_id=valid_token.user_id,
        team_id=team_object.team_id,
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
    )

    if (
        team_membership is None
        or team_membership.litellm_budget_table is None
        or not team_membership.litellm_budget_table.allowed_models
    ):
        return  # no per-member restriction — inherit team-level check

    member_allowed_models: Final[list[str]] = team_membership.litellm_budget_table.allowed_models
    try:
        _can_object_call_model(
            model=model,
            llm_router=llm_router,
            models=member_allowed_models,
            object_type="team",
            team_id=team_object.team_id,
        )
    except ProxyException:
        raise ProxyException(
            message=f"Team member not allowed to access model. User={valid_token.user_id}, Team={team_object.team_id}, Model={model}. Allowed member models = {member_allowed_models}",
            type=ProxyErrorTypes.team_model_access_denied,
            param="model",
            code=status.HTTP_403_FORBIDDEN,
        )


async def _team_max_budget_check(
    team_object: LiteLLM_TeamTable | None,
    valid_token: UserAPIKeyAuth | None,
    proxy_logging_obj: ProxyLogging,
):
    """
    Check if the team is over it's max budget.

    Raises:
        BudgetExceededError if the team is over it's max budget.
        Triggers a budget alert if the team is over it's max budget.
    """
    if team_object is not None and team_object.max_budget is not None:
        from litellm.proxy.proxy_server import get_current_spend

        # Read spend from cross-pod counter (Redis-first) or cached object (fallback)
        spend: Final = await get_current_spend(
            counter_key=f"spend:team:{team_object.team_id}",
            fallback_spend=team_object.spend or 0.0,
            max_budget=team_object.max_budget,
        )

        if math.isfinite(team_object.max_budget) and spend > team_object.max_budget:
            if valid_token:
                call_info: Final = CallInfo(
                    token=valid_token.token,
                    spend=spend,
                    max_budget=team_object.max_budget,
                    user_id=valid_token.user_id,
                    team_id=valid_token.team_id,
                    team_alias=valid_token.team_alias,
                    organization_id=valid_token.org_id,
                    event_group=Litellm_EntityType.TEAM,
                )
                asyncio.create_task(
                    proxy_logging_obj.budget_alerts(
                        type="team_budget",
                        user_info=call_info,
                    )
                )

            raise litellm.BudgetExceededError(
                current_cost=spend,
                max_budget=team_object.max_budget,
                message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}",
                entity_type=Litellm_EntityType.TEAM.value,
                entity_id=team_object.team_id,
            )


async def _team_multi_budget_check(
    team_object: LiteLLM_TeamTable | None,
):
    """
    Raises BudgetExceededError if any budget window in team_object.budget_limits is exceeded.

    Each window has its own Redis counter keyed by spend:team:{team_id}:window:{budget_duration}.
    Using budget_duration (not list index) keeps counters stable when windows are reordered
    or removed during a team update.
    """
    if team_object is None or not team_object.budget_limits:
        return

    from litellm.proxy.proxy_server import get_current_spend

    for window in team_object.budget_limits:
        w: dict = window if isinstance(window, dict) else window.model_dump()
        counter_key = f"spend:team:{team_object.team_id}:window:{w['budget_duration']}"
        window_spend = await get_current_spend(
            counter_key=counter_key,
            fallback_spend=0.0,
            max_budget=w["max_budget"],
            window_entity_type="Team",
            window_entity_id=team_object.team_id,
            window_start=get_budget_window_start(w),
        )
        if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]:
            raise litellm.BudgetExceededError(
                current_cost=window_spend,
                max_budget=w["max_budget"],
                message=(
                    f"ExceededBudget: Team={team_object.team_id} over {w['budget_duration']} budget. "
                    f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
                ),
                entity_type=Litellm_EntityType.TEAM.value,
                entity_id=team_object.team_id,
            )


async def _team_soft_budget_check(
    team_object: LiteLLM_TeamTable | None,
    valid_token: UserAPIKeyAuth | None,
    proxy_logging_obj: ProxyLogging,
):
    """
    Triggers a budget alert if the team is over it's soft budget.
    """
    if (
        team_object is not None
        and team_object.soft_budget is not None
        and team_object.spend is not None
        and team_object.spend >= team_object.soft_budget
    ):
        verbose_proxy_logger.debug(
            "Crossed Soft Budget for team %s, spend %s, soft_budget %s",
            team_object.team_id,
            team_object.spend,
            team_object.soft_budget,
        )
        if valid_token:
            # Extract alert emails from team metadata
            alert_emails: list[str] | None = None
            if team_object.metadata is not None and isinstance(team_object.metadata, dict):
                soft_budget_alert_emails: Final = team_object.metadata.get("soft_budget_alerting_emails")
                if soft_budget_alert_emails is not None:
                    if isinstance(soft_budget_alert_emails, list):
                        alert_emails = [
                            email for email in soft_budget_alert_emails if isinstance(email, str) and email.strip()
                        ]
                    elif isinstance(soft_budget_alert_emails, str):
                        # Handle comma-separated string
                        alert_emails = [email.strip() for email in soft_budget_alert_emails.split(",") if email.strip()]
                    # Filter out empty strings
                    if alert_emails:
                        alert_emails = [email for email in alert_emails if email]
                    else:
                        alert_emails = None

            # Only send team soft budget alerts if alert_emails are configured
            # Team soft budget alerts are sent via metadata.soft_budget_alerting_emails, not global alerting
            if alert_emails is None or len(alert_emails) == 0:
                verbose_proxy_logger.debug(
                    "Skipping team soft budget alert for team %s: no alert_emails configured in metadata.soft_budget_alerting_emails",
                    team_object.team_id,
                )
                return

            call_info: Final = CallInfo(
                token=valid_token.token,
                spend=team_object.spend,
                max_budget=team_object.max_budget,
                soft_budget=team_object.soft_budget,
                user_id=valid_token.user_id,
                team_id=valid_token.team_id,
                team_alias=valid_token.team_alias,
                organization_id=valid_token.org_id,
                user_email=None,  # Team-level alert, no specific user email
                key_alias=valid_token.key_alias,
                event_group=Litellm_EntityType.TEAM,
                alert_emails=alert_emails,
            )

            asyncio.create_task(
                proxy_logging_obj.budget_alerts(
                    type="soft_budget",
                    user_info=call_info,
                )
            )


async def _project_max_budget_check(
    project_object: LiteLLM_ProjectTableCachedObj | None,
    valid_token: UserAPIKeyAuth | None,
    proxy_logging_obj: ProxyLogging,
):
    """
    Check if the project is over its max budget.

    Raises:
        BudgetExceededError if the project is over its max budget.
        Triggers a budget alert if the project is over its max budget.
    """
    if project_object is None:
        return

    max_budget = None
    if project_object.litellm_budget_table is not None:
        max_budget = project_object.litellm_budget_table.max_budget

    if (
        max_budget is not None
        and project_object.spend is not None
        and math.isfinite(max_budget)
        and project_object.spend > max_budget
    ):
        if valid_token:
            call_info: Final = CallInfo(
                token=valid_token.token,
                spend=project_object.spend,
                max_budget=max_budget,
                user_id=valid_token.user_id,
                team_id=valid_token.team_id,
                team_alias=valid_token.team_alias,
                organization_id=valid_token.org_id,
                event_group=Litellm_EntityType.PROJECT,
            )
            asyncio.create_task(
                proxy_logging_obj.budget_alerts(
                    type="project_budget",
                    user_info=call_info,
                )
            )

        raise litellm.BudgetExceededError(
            current_cost=project_object.spend,
            max_budget=max_budget,
            message=f"Budget has been exceeded! Project={project_object.project_id} Current cost: {project_object.spend}, Max budget: {max_budget}",
            entity_type=Litellm_EntityType.PROJECT.value,
            entity_id=project_object.project_id,
        )


async def _project_soft_budget_check(
    project_object: LiteLLM_ProjectTableCachedObj | None,
    valid_token: UserAPIKeyAuth | None,
    proxy_logging_obj: ProxyLogging,
):
    """
    Triggers a budget alert if the project is over its soft budget.

    Mirrors _team_soft_budget_check() pattern.
    """
    if project_object is None:
        return

    soft_budget = None
    if project_object.litellm_budget_table is not None:
        soft_budget = project_object.litellm_budget_table.soft_budget

    if soft_budget is not None and project_object.spend is not None and project_object.spend >= soft_budget:
        verbose_proxy_logger.debug(
            "Crossed Soft Budget for project %s, spend %s, soft_budget %s",
            project_object.project_id,
            project_object.spend,
            soft_budget,
        )
        if valid_token:
            call_info: Final = CallInfo(
                token=valid_token.token,
                spend=project_object.spend,
                max_budget=None,
                soft_budget=soft_budget,
                user_id=valid_token.user_id,
                team_id=valid_token.team_id,
                team_alias=valid_token.team_alias,
                organization_id=valid_token.org_id,
                event_group=Litellm_EntityType.PROJECT,
            )
            asyncio.create_task(
                proxy_logging_obj.budget_alerts(
                    type="soft_budget",
                    user_info=call_info,
                )
            )


def _project_cache_key(project_id: str) -> str:
    return f"project_id:{project_id}"


async def get_project_object(
    project_id: str,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging | None = None,
) -> LiteLLM_ProjectTableCachedObj | None:
    """
    Fetch project object from cache or DB.

    Follows get_team_object() caching pattern with TTL and last_refreshed_at.

    Returns LiteLLM_ProjectTableCachedObj or None if not found.
    """
    if prisma_client is None:
        return None

    # Check cache first
    cache_key: Final = _project_cache_key(project_id)
    deserialized_project: Final = await user_api_key_cache.async_get_cache(
        key=cache_key,
        model_type=LiteLLM_ProjectTableCachedObj,
    )
    if deserialized_project is not None:
        return deserialized_project

    # Fetch from DB
    project_row: Final = await _model_dump_table(ProjectRepository(prisma_client)).find_unique(
        where={"project_id": project_id},
        include={"litellm_budget_table": True},
    )
    if project_row is None:
        return None

    project_obj: Final = LiteLLM_ProjectTableCachedObj.model_validate(project_row.model_dump())

    # Cache with TTL following _cache_management_object pattern
    project_obj.last_refreshed_at = time.time()
    await _cache_management_object(
        key=cache_key,
        value=project_obj,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
        model_type=LiteLLM_ProjectTableCachedObj,
    )

    return project_obj


async def delete_cached_project_object(
    project_id: str,
    user_api_key_cache: UserApiKeyCache,
) -> None:
    """
    Every endpoint that mutates litellm_projecttable must call this, or a stale project (e.g. a
    pre-update empty model allowlist) keeps being enforced until the TTL expires (LIT-3803).
    """
    from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast

    await evict_and_broadcast(
        cache_keys=(_project_cache_key(project_id),),
        user_api_key_cache=user_api_key_cache,
    )


async def _organization_max_budget_check(
    valid_token: UserAPIKeyAuth | None,
    team_object: LiteLLM_TeamTable | None,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging,
):
    """
    Check if the organization is over its max budget.

    This function checks the organization budget using:
    1. First, tries to use valid_token.org_id (if key has organization_id set)
    2. Falls back to team_object.organization_id (if key doesn't have org_id but team does)

    This ensures organization budget checks work even when keys don't have organization_id
    set directly, as long as their team belongs to an organization.

    Raises:
        BudgetExceededError if the organization is over its max budget.
        Triggers a budget alert if the organization is over its max budget.
    """
    if valid_token is None or prisma_client is None:
        return

    # Determine organization_id: first try from token, then fallback to team
    org_id: str | None = None
    if valid_token.org_id is not None:
        org_id = valid_token.org_id
    elif team_object is not None and team_object.organization_id is not None:
        org_id = team_object.organization_id

    # If no organization_id found, skip the check
    if org_id is None:
        return

    # Get organization object with budget table - use get_org_object so it can be mocked in tests
    try:
        org_table: Final = await get_org_object(
            org_id=org_id,
            prisma_client=prisma_client,
            user_api_key_cache=user_api_key_cache,
            proxy_logging_obj=proxy_logging_obj,
            include_budget_table=True,
        )
    except Exception:
        # If organization lookup fails, skip the check
        return

    if org_table is None:
        return

    # Get max_budget from organization's budget table
    org_max_budget: float | None = None
    if org_table.litellm_budget_table is not None:
        org_max_budget = org_table.litellm_budget_table.max_budget

    # Only check if organization has a valid max_budget set
    if org_max_budget is None or org_max_budget <= 0:
        return

    # Read spend from cross-pod counter (Redis-first) or cached object (fallback)
    from litellm.proxy.proxy_server import get_current_spend

    org_spend: Final = await get_current_spend(
        counter_key=f"spend:org:{org_id}",
        fallback_spend=org_table.spend or 0.0,
        max_budget=org_max_budget,
    )

    # Check if organization spend exceeds max budget
    if math.isfinite(org_max_budget) and org_spend >= org_max_budget:
        # Trigger budget alert
        call_info: Final = CallInfo(
            token=valid_token.token,
            spend=org_spend,
            max_budget=org_max_budget,
            user_id=valid_token.user_id,
            team_id=valid_token.team_id,
            team_alias=valid_token.team_alias,
            organization_id=org_id,
            event_group=Litellm_EntityType.ORGANIZATION,
        )
        asyncio.create_task(
            proxy_logging_obj.budget_alerts(
                type="organization_budget",
                user_info=call_info,
            )
        )

        raise litellm.BudgetExceededError(
            current_cost=org_spend,
            max_budget=org_max_budget,
            message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_spend}, Max budget: {org_max_budget}",
            entity_type=Litellm_EntityType.ORGANIZATION.value,
            entity_id=org_id,
        )


async def _tag_max_budget_check(
    request_body: dict,
    prisma_client: PrismaClient | None,
    user_api_key_cache: UserApiKeyCache,
    proxy_logging_obj: ProxyLogging,
    valid_token: UserAPIKeyAuth | None,
):
    """
    Check if any tags in the request are over their max budget.

    Raises:
        BudgetExceededError if any tag is over its max budget.
        Triggers a budget alert if any tag is over its max budget.
    """
    from litellm.proxy.common_utils.http_parsing_utils import get_tags_from_request_body

    if prisma_client is None:
        return

    # Get tags from request metadata
    tags: Final = get_tags_from_request_body(request_body=request_body)
    if not tags:
        return

    # Batch fetch all tags in one go
    tag_objects: Final = await get_tag_objects_batch(
        tag_names=tags,
        prisma_client=prisma_client,
        user_api_key_cache=user_api_key_cache,
        proxy_logging_obj=proxy_logging_obj,
    )

    # Check budget for each tag
    for tag_name in tags:
        tag_object = tag_objects.get(tag_name)
        if tag_object is None:
            continue

        # Check if tag has budget limits
        if tag_object.litellm_budget_table is not None and tag_object.litellm_budget_table.max_budget is not None:
            from litellm.proxy.proxy_server import get_current_spend

            tag_spend = await get_current_spend(
                counter_key=f"spend:tag:{tag_name}",
                fallback_spend=tag_object.spend or 0.0,
                max_budget=tag_object.litellm_budget_table.max_budget,
                fallback_authoritative=True,
            )
            if tag_spend <= tag_object.litellm_budget_table.max_budget:
                continue
            raise litellm.BudgetExceededError(
                current_cost=tag_spend,
                max_budget=tag_object.litellm_budget_table.max_budget,
                message=f"Budget has been exceeded! Tag={tag_name} Current cost: {tag_spend}, Max budget: {tag_object.litellm_budget_table.max_budget}",
                entity_type=Litellm_EntityType.TAG.value,
                entity_id=tag_name,
            )


def is_model_allowed_by_pattern(model: str, allowed_model_pattern: str) -> bool:
    """
    Check if a model matches an allowed pattern.
    Handles exact matches and wildcard patterns.

    Args:
        model (str): The model to check (e.g., "bedrock/anthropic.claude-3-5-sonnet-20240620")
        allowed_model_pattern (str): The allowed pattern (e.g., "bedrock/*", "*", "openai/*")

    Returns:
        bool: True if model matches the pattern, False otherwise
    """
    if "*" in allowed_model_pattern:
        pattern: Final = f"^{allowed_model_pattern.replace('*', '.*')}$"
        return bool(re.match(pattern, model))

    return False


def _model_matches_any_wildcard_pattern_in_list(model: str, allowed_model_list: list) -> bool:
    """
    Returns True if a model matches any wildcard pattern in a list.

    eg.
    - model=`bedrock/us.amazon.nova-micro-v1:0`, allowed_models=`bedrock/*` returns True
    - model=`bedrock/us.amazon.nova-micro-v1:0`, allowed_models=`bedrock/us.*` returns True
    - model=`bedrockzzzz/us.amazon.nova-micro-v1:0`, allowed_models=`bedrock/*` returns False
    """

    if any(
        _is_wildcard_pattern(allowed_model_pattern)
        and is_model_allowed_by_pattern(model=model, allowed_model_pattern=allowed_model_pattern)
        for allowed_model_pattern in allowed_model_list
    ):
        return True

    if any(
        _is_wildcard_pattern(allowed_model_pattern)
        and _model_custom_llm_provider_matches_wildcard_pattern(
            model=model, allowed_model_pattern=allowed_model_pattern
        )
        for allowed_model_pattern in allowed_model_list
    ):
        return True

    return False


def _model_custom_llm_provider_matches_wildcard_pattern(model: str, allowed_model_pattern: str) -> bool:
    """
    Returns True for this scenario:
    - `model=gpt-4o`
    - `allowed_model_pattern=openai/*`

    or
    - `model=claude-3-5-sonnet-20240620`
    - `allowed_model_pattern=anthropic/*`

    A model that already carries a namespace get_llm_provider did not consume
    (e.g. `bedrockz/anthropic.claude-...`) is never granted here: its provider was
    inferred from a fragment of the full string, so rebuilding
    `{provider}/{model}` would produce `bedrock/bedrockz/...` and slip an
    unrecognized namespace through a `bedrock/*` key.
    """
    try:
        stripped_model, custom_llm_provider, _, _ = get_llm_provider(model=model)
    except Exception:
        return False

    if stripped_model == model and "/" in model:
        return False

    return is_model_allowed_by_pattern(
        model=f"{custom_llm_provider}/{stripped_model}",
        allowed_model_pattern=allowed_model_pattern,
    )


def _is_wildcard_pattern(allowed_model_pattern: str) -> bool:
    """
    Returns True if the pattern is a wildcard pattern.

    Checks if `*` is in the pattern.
    """
    return "*" in allowed_model_pattern


async def vector_store_access_check(
    request_body: dict,
    team_object: LiteLLM_TeamTable | None,
    valid_token: UserAPIKeyAuth | None,
):
    """
    Checks if the object (key, team, org) has access to the vector store.

    Raises ProxyException if the object (key, team, org) cannot access the specific vector store.
    """
    from litellm.proxy.proxy_server import prisma_client

    #########################################################
    # Get the vector store the user is trying to access
    #########################################################
    if prisma_client is None:
        verbose_proxy_logger.debug("Prisma client not found, skipping vector store access check")
        return True

    if litellm.vector_store_registry is None:
        verbose_proxy_logger.debug("Vector store registry not found, skipping vector store access check")
        return True

    vector_store_ids_to_run: Final = litellm.vector_store_registry.get_vector_store_ids_to_run(
        non_default_params=request_body, tools=request_body.get("tools", None)
    )
    if vector_store_ids_to_run is None:
        verbose_proxy_logger.debug("Vector store to run not found, skipping vector store access check")
        return True

    #########################################################
    # Check if the object (key, team, org) has access to the vector store
    #########################################################
    # Check if the key can access the vector store
    if valid_token is not None and valid_token.object_permission_id is not None:
        key_object_permission: Final = await _object_permission_table(
            ObjectPermissionRepository(prisma_client)
        ).find_unique(
            where={"object_permission_id": valid_token.object_permission_id},
        )
        if key_object_permission is not None:
            _can_object_call_vector_stores(
                object_type="key",
                vector_store_ids_to_run=vector_store_ids_to_run,
                object_permissions=key_object_permission,
            )

    # Check if the team can access the vector store
    if team_object is not None and team_object.object_permission_id is not None:
        team_object_permission: Final = await _object_permission_table(
            ObjectPermissionRepository(prisma_client)
        ).find_unique(
            where={"object_permission_id": team_object.object_permission_id},
        )
        if team_object_permission is not None:
            _can_object_call_vector_stores(
                object_type="team",
                vector_store_ids_to_run=vector_store_ids_to_run,
                object_permissions=team_object_permission,
            )
    return True


def _can_object_call_vector_stores(
    object_type: Literal["key", "team", "org"],
    vector_store_ids_to_run: list[str],
    object_permissions: LiteLLM_ObjectPermissionTable | None,
):
    """
    Raises ProxyException if the object (key, team, org) cannot access the specific vector store.
    """
    if object_permissions is None:
        return True

    if object_permissions.vector_stores is None:
        return True

    # If length is 0, then the object has access to all vector stores.
    if len(object_permissions.vector_stores) == 0:
        return True

    for vector_store_id in vector_store_ids_to_run:
        if vector_store_id not in object_permissions.vector_stores:
            raise ProxyException(
                message=f"User not allowed to access vector store. Tried to access {vector_store_id}. Only allowed to access {object_permissions.vector_stores}",
                type=ProxyErrorTypes.get_vector_store_access_error_type_for_object(object_type),
                param="vector_store",
                code=status.HTTP_401_UNAUTHORIZED,
            )

    return True