diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 751f855ea34..aeb52a3d239 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2397,6 +2397,17 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): "`statement_cache_size`). Keys here override any default LiteLLM sets." ), ) + database_disable_prepared_statements: Optional[bool] = Field( + None, + description=( + "Disable server-side prepared statements by setting Prisma's " + "`pgbouncer=true` URL param. Use this for pgbouncer transaction-pooling " + "deployments, or to prevent the 'cached plan must not change result " + "type' error that pooled connections hit during rolling schema " + "migrations. An explicit `pgbouncer` in `database_extra_connection_params` " + "takes precedence." + ), + ) database_type: Optional[Literal["dynamo_db"]] = Field( None, description="to use dynamodb instead of postgres db" ) @@ -2533,6 +2544,24 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="List of MCP server fields that must be filled in for a submission to pass standards checks (e.g. ['description', 'source_url', 'alias']).", ) + disable_budget_reservation: Optional[bool] = Field( + None, + description=( + "If True, disables the optimistic per-request budget reservation " + "introduced in v1.84.0. " + "WARNING: This weakens hard budget enforcement. Without the reservation, " + "a burst of concurrent requests from a single key can each pass the " + "read-time spend check before any of them is charged, allowing a " + "configured budget to be exceeded under high concurrency. " + "Budgets are still evaluated on every request at read time, so " + "an already-exhausted budget is still rejected. " + "Enable only if your deployment is experiencing phantom " + "BudgetExceededError responses caused by leaked reservations " + "(see GitHub issue #27639). " + "A proxy-level WARNING is logged on every request while this flag " + "is active as a reminder that hard enforcement is relaxed." + ), + ) class ConfigYAML(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/auth/auth_exception_handler.py b/litellm/proxy/auth/auth_exception_handler.py index 431db4254eb..f276a1350f1 100644 --- a/litellm/proxy/auth/auth_exception_handler.py +++ b/litellm/proxy/auth/auth_exception_handler.py @@ -134,6 +134,16 @@ class UserAPIKeyAuthExceptionHandler: ) elif isinstance(e, ProxyException): raise e + if PrismaDBExceptionHandler.is_database_service_unavailable_error(e): + raise ProxyException( + message=( + "Service Unavailable, the authentication database is " + "temporarily unreachable. Please retry shortly." + ), + type=ProxyErrorTypes.no_db_connection, + param="None", + code=status.HTTP_503_SERVICE_UNAVAILABLE, + ) raise ProxyException( message="Authentication Error, " + str(e), type=ProxyErrorTypes.auth_error, diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 2e6cd1f8e70..37809e043a8 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -2108,6 +2108,7 @@ async def _run_centralized_common_checks( # noqa: PLR0915 user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, skip_budget_checks=skip_budget_checks, + general_settings=general_settings, ) @@ -2128,12 +2129,23 @@ async def _reserve_budget_after_common_checks( user_api_key_cache: UserApiKeyCache, proxy_logging_obj: ProxyLogging, skip_budget_checks: bool, + general_settings: dict, end_user_id: Optional[str] = None, end_user_object: Optional[LiteLLM_EndUserTable] = None, ) -> None: user_api_key_auth_obj.budget_reservation = None if skip_budget_checks: return + if general_settings.get("disable_budget_reservation") is True: + verbose_proxy_logger.warning( + "disable_budget_reservation is enabled: skipping optimistic budget " + "reservation. Budget enforcement is read-time only — concurrent " + "requests can each pass the spend check before their cost is recorded, " + "so a configured budget may be briefly exceeded under high concurrency. " + "Set disable_budget_reservation to False or remove it to restore " + "hard per-request budget enforcement." + ) + return from litellm.proxy.spend_tracking.budget_reservation import ( reserve_budget_for_request, diff --git a/litellm/proxy/db/exception_handler.py b/litellm/proxy/db/exception_handler.py index ab9d341aa51..c500e727595 100644 --- a/litellm/proxy/db/exception_handler.py +++ b/litellm/proxy/db/exception_handler.py @@ -109,6 +109,92 @@ class PrismaDBExceptionHandler: return True return False + @staticmethod + def is_prisma_engine_internal_error(e: Exception) -> bool: + """True iff ``e`` is a non-``PrismaError`` exception raised from inside + prisma-client-py's query-engine layer. + + During the instant a DB connection is torn down, the query engine can + return a malformed error payload (``user_facing_error.meta`` is + ``null``). prisma-client-py's ``handle_response_errors`` then crashes + with ``AttributeError: 'NoneType' object has no attribute 'get'`` + before it can raise the proper P1001 "can't reach database server" + error. That AttributeError carries no connection keyword, so it can't + be matched by message; identify it by its ``prisma.engine`` origin + instead. + + Recognized ``PrismaError`` subclasses are excluded: connectivity ones + are already classified by type/keyword above, and data-layer ones + (the DB IS reachable) must stay 401. + """ + import prisma + + if isinstance(e, prisma.errors.PrismaError): + return False + tb = getattr(e, "__traceback__", None) + while tb is not None: + if tb.tb_frame.f_globals.get("__name__", "").startswith("prisma.engine"): + return True + tb = tb.tb_next + return False + + @staticmethod + def is_database_service_unavailable_error(e: Exception) -> bool: + """True iff the exception means the database could not answer at the + infrastructure level (connection refused, socket/interface failure, + timeout) rather than a genuine auth failure (key not found) or a + data-layer error (the DB IS reachable and rejected the data). + + Auth must answer 401 only for a key the DB confirms is invalid. When + the DB itself is unreachable, the request has to surface as 503 so + callers retry instead of treating valid keys as invalid during an + outage. + + Note: prisma-client-py mislabels the P1001 "can't reach database + server" connectivity failure as a ``DataError`` (a data-layer type), + so a type-only check misses real outages. ``is_database_transport_error`` + keyword-matches the connection message and catches that masquerade, + while genuine data errors (no connection keyword) correctly stay 401. + + The Postgres "cached plan must not change result type" error is matched + here, not in ``is_database_transport_error``: it is a transient stale-DB- + state condition (not an invalid key), but the connection is healthy so it + must not trigger a reconnect. + + A non-``PrismaError`` raised from inside the prisma query engine (e.g. + the ``AttributeError`` from ``handle_response_errors`` when the engine + returns a malformed error payload mid-tear-down) is also treated as + unavailable; see ``is_prisma_engine_internal_error``. + """ + import asyncio + + if PrismaDBExceptionHandler.is_database_connection_error(e): + return True + if PrismaDBExceptionHandler.is_database_transport_error(e): + return True + if PrismaDBExceptionHandler.is_prisma_engine_internal_error(e): + return True + if "cached plan must not change result type" in str(e).lower(): + return True + + # OSError already covers ConnectionError and (Py3.3+) TimeoutError. + # asyncio.TimeoutError is a distinct class before Py3.11. + if isinstance(e, (OSError, asyncio.TimeoutError)): + return True + + try: + import asyncpg + except ImportError: + return False + + return isinstance( + e, + ( + asyncpg.exceptions.PostgresConnectionError, + asyncpg.exceptions.InterfaceError, + ), + ) + @staticmethod def handle_db_exception(e: Exception): """ diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 722fcd30033..404ed4491e5 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -490,9 +490,45 @@ def _get_public_model_name( patch_data: updateDeployment, db_model: Deployment, ) -> str: - """Determine the public model name from patch or existing model.""" - if patch_data.model_name: - return patch_data.model_name + """Determine the public model name from patch or existing model. + + The top-level ``model_name`` is the rename channel. For team-scoped rows + the DB ``model_name`` column holds an internal routing key + (``model_name_{team_id}_{uuid}``), and ``/model/info`` historically leaked + it into the dashboard edit form, so a non-rename save (e.g. a TPM tweak) + would PATCH the internal name and the update path would treat it as a + rename -- overwriting ``team_public_model_name`` and rewriting the team ACL + (see issue #28382). + + Guard against that by ignoring an incoming ``model_name`` that matches the + internal shape, or is a no-op against the current DB column. Anything else + is a genuine rename and wins. We deliberately do NOT read + ``patch_data.model_info.team_public_model_name``: the dashboard passes the + existing ``model_info`` blob through untouched on a rename, so honoring it + would return the OLD public name and silently drop the rename. + + Precedence (highest first): + 1. patch_data.model_name -- a genuine rename: not internal-shape and not a + no-op against db_model.model_name. + 2. db_model.model_info.team_public_model_name -- existing public name. + 3. db_model.model_name -- last-resort fallback for legacy rows. + """ + team_id = (patch_data.model_info.team_id if patch_data.model_info else None) or ( + db_model.model_info.team_id if db_model.model_info else None + ) + + def _is_internal_shape(name: Optional[str]) -> bool: + if team_id is None or not name: + return False + return name.startswith(f"model_name_{team_id}_") + + incoming = patch_data.model_name + if ( + incoming + and not _is_internal_shape(incoming) + and incoming != db_model.model_name + ): + return incoming if db_model.model_info and db_model.model_info.team_public_model_name: return db_model.model_info.team_public_model_name diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py index a94672f9487..a912a88a993 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py @@ -100,6 +100,42 @@ class AnthropicPassthroughLoggingHandler: return get_end_user_id_from_request_body(request_body) return None + @staticmethod + def _resolve_costing_model(model: str, logging_obj: LiteLLMLoggingObj) -> str: + if model and model != "unknown": + return model + litellm_params = (getattr(logging_obj, "model_call_details", {}) or {}).get( + "litellm_params", {} + ) or {} + deployment_model = litellm_params.get("model") + if deployment_model and deployment_model != "unknown": + return deployment_model + model_group = (litellm_params.get("metadata", {}) or {}).get("model_group") + if model_group: + return model_group.removeprefix("passthrough/") + return model + + @staticmethod + def _extract_model_from_anthropic_chunks( + all_chunks: Sequence[Union[str, bytes]], + ) -> Optional[str]: + for raw in all_chunks: + text = raw.decode("utf-8") if isinstance(raw, bytes) else raw + for line in text.splitlines(): + if not line.startswith("data:"): + continue + try: + data = json.loads(line[len("data:") :].strip()) + except (json.JSONDecodeError, ValueError): + continue + if not isinstance(data, dict): + continue + if data.get("type") == "message_start": + model = (data.get("message") or {}).get("model") + if model: + return model + return None + @staticmethod def _create_anthropic_response_logging_payload( litellm_model_response: Union[ModelResponse, TextCompletionResponse], @@ -127,6 +163,10 @@ class AnthropicPassthroughLoggingHandler: "custom_llm_provider" ) + model = AnthropicPassthroughLoggingHandler._resolve_costing_model( + model, logging_obj + ) + # Prepend custom_llm_provider to model if not already present model_for_cost = model if custom_llm_provider and not model.startswith(f"{custom_llm_provider}/"): @@ -213,6 +253,15 @@ class AnthropicPassthroughLoggingHandler: ): model = cast(str, litellm_logging_obj.model_call_details.get("model")) + if not model or model == "unknown": + chunk_model = ( + AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks( + all_chunks + ) + ) + if chunk_model: + model = chunk_model + complete_streaming_response = ( AnthropicPassthroughLoggingHandler._build_complete_streaming_response( all_chunks=all_chunks, @@ -468,6 +517,13 @@ class AnthropicPassthroughLoggingHandler: # Process each individual event for event_str in individual_events: try: + # Skip OpenAI-style [DONE] sentinels some Anthropic-compatible + # providers emit. Match the whole SSE line so a valid chunk whose + # text payload happens to contain "[DONE]" is not dropped. + if any( + line.strip() == "data: [DONE]" for line in event_str.split("\n") + ): + continue transformed_openai_chunk = anthropic_model_response_iterator.convert_str_chunk_to_generic_chunk( chunk=event_str ) @@ -476,6 +532,14 @@ class AnthropicPassthroughLoggingHandler: except (StopIteration, StopAsyncIteration): break + except json.JSONDecodeError: + # Some upstreams emit non-JSON SSE lines; skip them so the + # logging pipeline is not broken by a single bad frame. + verbose_proxy_logger.debug( + "Skipping non-JSON SSE event: %s", + event_str[:200], + ) + continue complete_streaming_response = litellm.stream_chunk_builder( chunks=all_openai_chunks, diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index c0246f234a8..0d60806da54 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -44,15 +44,19 @@ def _build_db_connection_url_params( pool_timeout: Optional[Union[int, float]], connect_timeout: Optional[Union[int, float]] = None, socket_timeout: Optional[Union[int, float]] = None, + disable_prepared_statements: bool = False, extra_params: Optional[dict] = None, ) -> dict: """Build the Prisma DATABASE_URL query params controlling connection pool behavior. `connect_timeout` / `socket_timeout` map to the Prisma URL params of the same name (https://www.prisma.io/docs/orm/overview/databases/postgresql) and are - omitted when None so Prisma's defaults apply. `extra_params` is an - untyped passthrough — keys it provides win over the named arguments above, - so it can be used to override any default we set here. + omitted when None so Prisma's defaults apply. `disable_prepared_statements` + sets `pgbouncer=true`, which makes Prisma stop using server-side prepared + statements (pgbouncer transaction-pool compatible; also sidesteps the + "cached plan must not change result type" error during rolling migrations). + `extra_params` is an untyped passthrough — keys it provides win over the + named arguments above, so it can be used to override any default we set here. """ params: dict = { "connection_limit": connection_limit, @@ -63,6 +67,8 @@ def _build_db_connection_url_params( params["connect_timeout"] = connect_timeout if socket_timeout is not None: params["socket_timeout"] = socket_timeout + if disable_prepared_statements: + params["pgbouncer"] = "true" if extra_params: params.update(extra_params) return params @@ -925,6 +931,7 @@ def run_server( # noqa: PLR0915 db_connection_timeout: Optional[Union[int, float]] = 60 db_connect_timeout: Optional[Union[int, float]] = None db_socket_timeout: Optional[Union[int, float]] = None + db_disable_prepared_statements: bool = False db_extra_connection_params: Optional[dict] = None general_settings = {} ### GET DB TOKEN FOR IAM AUTH ### @@ -1045,6 +1052,17 @@ def run_server( # noqa: PLR0915 ) db_connect_timeout = general_settings.get("database_connect_timeout") db_socket_timeout = general_settings.get("database_socket_timeout") + _disable_prepared_statements = general_settings.get( + "database_disable_prepared_statements", False + ) + if isinstance(_disable_prepared_statements, str): + from litellm.secret_managers.main import str_to_bool + + db_disable_prepared_statements = ( + str_to_bool(_disable_prepared_statements) is True + ) + else: + db_disable_prepared_statements = bool(_disable_prepared_statements) db_extra_connection_params = general_settings.get( "database_extra_connection_params" ) @@ -1092,6 +1110,7 @@ def run_server( # noqa: PLR0915 pool_timeout=db_connection_timeout, connect_timeout=db_connect_timeout, socket_timeout=db_socket_timeout, + disable_prepared_statements=db_disable_prepared_statements, extra_params=db_extra_connection_params, ) if os.getenv("DATABASE_URL", None) is not None: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b296792cd09..810cb91fae4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -10907,16 +10907,26 @@ def get_direct_access_models( return direct_access_models -async def get_all_team_and_direct_access_models( +def _filter_models_to_user_accessible(all_models: List[Dict]) -> List[Dict]: + """Keep only deployments the caller can use via direct access or team membership.""" + return [ + _model + for _model in all_models + if _model.get("model_info", {}).get("direct_access", False) + or _model.get("model_info", {}).get("access_via_team_ids", []) + ] + + +async def _populate_team_access_on_models( user_api_key_dict: UserAPIKeyAuth, prisma_client: PrismaClient, llm_router: Router, all_models: List[Dict], ) -> List[Dict]: """ - Get all models across all teams user is in. + Populate `model_info.access_via_team_ids` and `model_info.direct_access` + without filtering the model list. """ - user_teams: Optional[Union[List[str], Literal["*"]]] = None direct_access_models: List[str] = [] if user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN: @@ -10935,7 +10945,6 @@ async def get_all_team_and_direct_access_models( user_db_object=user_object, llm_router=llm_router, ) - ## ADD ACCESS_VIA_TEAM_IDS TO ALL MODELS if user_teams is not None: team_models = await get_all_team_models( user_teams=user_teams, @@ -10958,23 +10967,33 @@ async def get_all_team_and_direct_access_models( model_id, [] ) - ## ADD DIRECT_ACCESS TO RELEVANT MODELS - + direct_access_model_ids = set(direct_access_models) for _model in all_models: model_id = _model.get("model_info", {}).get("id", None) - if model_id is not None and model_id in direct_access_models: - _model["model_info"]["direct_access"] = True + if model_id is not None: + _model["model_info"]["direct_access"] = model_id in direct_access_model_ids - ## FILTER OUT MODELS THAT ARE NOT IN DIRECT_ACCESS_MODELS OR ACCESS_VIA_TEAM_IDS - only show user models they can call - all_models = [ - _model - for _model in all_models - if _model.get("model_info", {}).get("direct_access", False) - or _model.get("model_info", {}).get("access_via_team_ids", []) - ] return all_models +async def get_all_team_and_direct_access_models( + user_api_key_dict: UserAPIKeyAuth, + prisma_client: PrismaClient, + llm_router: Router, + all_models: List[Dict], +) -> List[Dict]: + """ + Get all models across all teams user is in. + """ + all_models = await _populate_team_access_on_models( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + llm_router=llm_router, + all_models=all_models, + ) + return _filter_models_to_user_accessible(all_models) + + def _enrich_model_info_with_litellm_data( model: Dict[str, Any], debug: bool = False, llm_router: Optional[Router] = None ) -> Dict[str, Any]: @@ -11083,6 +11102,22 @@ async def _get_caller_byok_team_scope( return set(user_row.teams or []) +def _byok_row_outside_caller_teams( + model_info_dict: Dict[str, Any], allowed_team_ids: Optional[Set[str]] +) -> bool: + """Whether a team BYOK row belongs to a team the caller is not a member of. + + `team_id` is only set on team BYOK rows; non-team rows fall through + unaffected. `allowed_team_ids is None` means no scoping (e.g. admins). + """ + if allowed_team_ids is None: + return False + team_id = model_info_dict.get("team_id") + if team_id is None: + return False + return team_id not in allowed_team_ids + + # Hard cap on rows the DB-side BYOK search may pull when results need to be # sorted across the full match set. Without this, an authenticated caller # can hit `/v2/model/info?search=&sortBy=` and force the @@ -11204,15 +11239,7 @@ async def _apply_search_filter_to_models( ) def _is_byok_outside_caller_teams(model_info_dict: Dict[str, Any]) -> bool: - # `team_id` is only set on team BYOK rows. Non-team rows fall - # through unaffected — they are gated by other paths (router - # membership, direct_access, include_team_models). - if allowed_team_ids is None: - return False - team_id = model_info_dict.get("team_id") - if team_id is None: - return False - return team_id not in allowed_team_ids + return _byok_row_outside_caller_teams(model_info_dict, allowed_team_ids) def _model_matches_search(m: Dict[str, Any]) -> bool: # Team BYOK models persist an internal `model_name` @@ -11875,6 +11902,9 @@ async def model_info_v2( # Update total count to include agents search_total_count = len(all_models) + # Translate `model_name` to the public name for team-scoped rows. + all_models = [_translate_model_name_for_response(m) for m in all_models] + return _paginate_models_response( all_models=all_models, page=page, @@ -12309,6 +12339,99 @@ async def model_metrics_exceptions( return {"data": response, "exception_types": list(exception_types)} +def _deployment_matches_allowed_model_names( + model: Dict[str, Any], allowed_model_names: Set[str] +) -> bool: + """Match a router deployment against allowed public model names. + + Team-scoped rows store an internal routing key in ``model_name``; callers + with key/team restrictions still refer to the public name in + ``model_info.team_public_model_name``. + """ + if model.get("model_name") in allowed_model_names: + return True + model_info = model.get("model_info") + if not isinstance(model_info, dict): + return False + team_public_model_name = model_info.get("team_public_model_name") + return ( + isinstance(team_public_model_name, str) + and team_public_model_name in allowed_model_names + ) + + +def _get_v1_model_info_allowed_model_names( + user_api_key_dict: UserAPIKeyAuth, + llm_router: Router, +) -> Optional[Set[str]]: + """Return key/team allowlisted public model names, or None if unrestricted.""" + model_access_groups = llm_router.get_model_access_groups() + proxy_model_list = llm_router.get_model_names() + key_models = get_key_models( + user_api_key_dict=user_api_key_dict, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + ) + team_models = get_team_models( + team_models=user_api_key_dict.team_models, + proxy_model_list=proxy_model_list, + model_access_groups=model_access_groups, + ) + if not key_models and not team_models: + return None + return set( + get_complete_model_list( + key_models=key_models, + team_models=team_models, + proxy_model_list=proxy_model_list, + user_model=user_model, + infer_model_from_keys=general_settings.get("infer_model_from_keys", False), + llm_router=llm_router, + return_wildcard_routes=False, + ) + ) + + +def _filter_v1_model_info_deployments( + all_models: List[dict], + allowed_model_names: Optional[Set[str]], +) -> List[dict]: + if allowed_model_names is None: + return all_models + return [ + model + for model in all_models + if _deployment_matches_allowed_model_names(model, allowed_model_names) + ] + + +def _translate_model_name_for_response(model: dict) -> dict: + """For team-scoped DB rows, replace `model_name` with the public name + in `model_info.team_public_model_name` before returning. The DB column + and the in-memory router index keep the internal mangled name + (`model_name_{team_id}_{uuid}`) as the routing key -- this swap is a + presentation-layer concern. Returns a shallow copy; never mutates. + + Without this swap the internal name leaks into `/v1/model/info` and + `/v2/model/info`, the dashboard binds its edit form to it, and a + non-rename save round-trips the internal name back -- corrupting + `team_public_model_name` and the team ACL (see issue #28382). + """ + if not isinstance(model, dict): + return model + model_info = model.get("model_info") or {} + if not isinstance(model_info, dict): + return model + team_public = model_info.get("team_public_model_name") + team_id = model_info.get("team_id") + if not team_public or not team_id: + return model + current = model.get("model_name") or "" + if not current.startswith(f"model_name_{team_id}_"): + return model + return {**model, "model_name": team_public} + + def _get_proxy_model_info(model: dict) -> dict: # provided model_info in config.yaml model_info = model.get("model_info", {}) @@ -12349,7 +12472,7 @@ def _get_proxy_model_info(model: dict) -> dict: deployment_dict=model, excluded_keys={"litellm_credential_name"} ) - return model + return _translate_model_name_for_response(model) @router.get( @@ -12365,6 +12488,14 @@ def _get_proxy_model_info(model: dict) -> dict: async def model_info_v1( # noqa: PLR0915 user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), litellm_model_id: Optional[str] = None, + include_team_models: Optional[bool] = fastapi.Query( + False, + description="When true, filter to deployments the caller can use via direct access or team membership.", + ), + teamId: Optional[str] = fastapi.Query( + None, + description="Filter models by team ID. Returns models with direct_access=True or teamId in access_via_team_ids", + ), ): """ Provides more info about each model in /models, including config.yaml descriptions (except api key and api base) @@ -12374,6 +12505,11 @@ async def model_info_v1( # noqa: PLR0915 - When litellm_model_id is passed, it will return the info for that specific model - When litellm_model_id is not passed, it will return the info for all models + - include_team_models: When true, filter to deployments the caller can use (same as /v2/model/info). + - teamId: Filter to models accessible by the given team. + + Each model in the list response includes `model_info.access_via_team_ids` and + `model_info.direct_access` when the proxy database is connected. Returns: Returns a dictionary containing information about each model. @@ -12400,6 +12536,12 @@ async def model_info_v1( # noqa: PLR0915 """ global llm_model_list, general_settings, user_config_file_path, proxy_config, llm_router, user_model + # Unit tests call this handler directly; FastAPI normally resolves Query defaults. + if not isinstance(include_team_models, bool): + include_team_models = False + if not isinstance(teamId, str): + teamId = None + if user_model is not None: # user is trying to get specific model from litellm router try: @@ -12436,6 +12578,14 @@ async def model_info_v1( # noqa: PLR0915 }, ) + if prisma_client is None and ( + include_team_models or (teamId is not None and teamId.strip()) + ): + raise HTTPException( + status_code=500, + detail={"error": CommonProxyErrors.db_not_connected_error.value}, + ) + if litellm_model_id is not None: # user is trying to get specific model from litellm router deployment_info = llm_router.get_deployment(model_id=litellm_model_id) @@ -12449,48 +12599,82 @@ async def model_info_v1( # noqa: PLR0915 _deployment_info_dict = _get_proxy_model_info( model=deployment_info.model_dump(exclude_none=True) ) - return {"data": [_deployment_info_dict]} + single_model_list: List[dict] = [_deployment_info_dict] + if prisma_client is not None: + single_model_list = await _populate_team_access_on_models( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + llm_router=llm_router, + all_models=single_model_list, + ) + if include_team_models: + single_model_list = _filter_models_to_user_accessible(single_model_list) + if teamId is not None and teamId.strip(): + single_model_list = await _filter_models_by_team_id( + all_models=single_model_list, + team_id=teamId.strip(), + prisma_client=prisma_client, + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + ) + return {"data": single_model_list} - all_models: List[dict] = [] - model_access_groups: Dict[str, List[str]] = defaultdict(list) - ## CHECK IF MODEL RESTRICTIONS ARE SET AT KEY/TEAM LEVEL ## - if llm_router is None: - proxy_model_list = [] - else: - proxy_model_list = llm_router.get_model_names() - model_access_groups = llm_router.get_model_access_groups() - key_models = get_key_models( + # Return router deployments (same source as /v2/model/info), not wildcard- + # expanded model names from get_complete_model_list(). Team-scoped rows + # use internal routing keys (model_name_{team_id}_{uuid}) and were omitted + # when v1 resolved models only via public model_name strings. + all_models: List[dict] = copy.deepcopy(llm_router.model_list) + allowed_model_names = _get_v1_model_info_allowed_model_names( user_api_key_dict=user_api_key_dict, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - ) - team_models = get_team_models( - team_models=user_api_key_dict.team_models, - proxy_model_list=proxy_model_list, - model_access_groups=model_access_groups, - ) - all_models_str = get_complete_model_list( - key_models=key_models, - team_models=team_models, - proxy_model_list=proxy_model_list, - user_model=user_model, - infer_model_from_keys=general_settings.get("infer_model_from_keys", False), llm_router=llm_router, ) - if len(all_models_str) > 0: - _relevant_models = [] - for model in all_models_str: - router_models = llm_router.get_model_list(model_name=model) - if router_models is not None: - _relevant_models.extend(router_models) - if llm_model_list is not None: - all_models = copy.deepcopy(_relevant_models) # type: ignore - else: - all_models = [] + all_models = _filter_v1_model_info_deployments( + all_models=all_models, + allowed_model_names=allowed_model_names, + ) - for in_place_model in all_models: - in_place_model = _get_proxy_model_info(model=in_place_model) + # Team BYOK deployments carry an internal routing key and other teams' + # public name/team_id/api_base; drop the ones the caller cannot access so + # listing the full router model_list does not leak cross-team metadata. + allowed_team_ids = await _get_caller_byok_team_scope( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + ) + all_models = [ + model + for model in all_models + if not _byok_row_outside_caller_teams( + model.get("model_info") or {}, allowed_team_ids + ) + ] + + if prisma_client is not None: + all_models = await _populate_team_access_on_models( + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, + llm_router=llm_router, + all_models=all_models, + ) + + if include_team_models: + all_models = _filter_models_to_user_accessible(all_models) + + all_models = [ + _translate_model_name_for_response( + _enrich_model_info_with_litellm_data(model=model, llm_router=llm_router) + ) + for model in all_models + ] + + if teamId is not None and teamId.strip(): + all_models = await _filter_models_by_team_id( + all_models=all_models, + team_id=teamId.strip(), + prisma_client=cast(PrismaClient, prisma_client), + llm_router=llm_router, + user_api_key_dict=user_api_key_dict, + ) verbose_proxy_logger.debug("all_models: %s", all_models) return {"data": all_models} diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 0e72f47e224..0720389e1f2 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -3159,40 +3159,49 @@ class PrismaClient: self, sql_query: str, *args ) -> Optional[dict]: """ - Execute a query with automatic fallback for PostgreSQL cached plan errors. + Execute a query, recovering once from PostgreSQL's "cached plan must not + change result type" error. - This handles the "cached plan must not change result type" error that occurs - during rolling deployments when schema changes are applied while old pods - still have cached query plans expecting the old schema. + That error surfaces during rolling deployments when a schema change + invalidates the prepared-statement plans that pooled connections still + hold. Clearing only the server-side plans with DEALLOCATE ALL makes + things worse: Prisma's query engine keeps a per-connection client-side + cache of prepared-statement names, so once the server drops a plan the + engine re-sends a name PostgreSQL no longer recognizes and the + connection breaks with `prepared statement "sN" does not exist`. With a + small pool that connection stays poisoned and every auth lookup fails. - Args: - sql_query: SQL query string to execute + Recreating the Prisma client kills the engine subprocess and drops the + server-side plans and the engine's client-side name cache together, so + the retried query is prepared fresh. We reconnect through + `attempt_db_reconnect`, which is singleflight: when a schema change + poisons every pooled connection at once, the first cached-plan error + recreates the client and the concurrent waiters reuse that single + recreate instead of racing to kill each other's fresh engine. We then + retry the identical query exactly once. - Returns: - Query result or None + The retry reuses the original query byte-for-byte. Mutating the SQL + (e.g. injecting a unique comment) would defeat PostgreSQL's plan cache, + forcing a fresh plan on every request and pegging the database CPU. - Raises: - Original exception if not a cached plan error + If the reconnect is skipped because a recent reconnect is still within + its cooldown, the retry runs against the same connection and may fail + again; the get_data backoff decorator re-runs the lookup and a later + attempt reconnects once the cooldown elapses. """ try: return await self.db.query_first(sql_query, *args) except Exception as e: - error_str = str(e) - if "cached plan must not change result type" in error_str: - # Force PostgreSQL to re-plan by invalidating the cache - # Add a unique comment to make the query different - sql_query_retry = sql_query.replace( - "SELECT", - f"SELECT /* cache_invalidated_{int(time.time() * 1000)} */", - ) - verbose_proxy_logger.warning( - "PostgreSQL cached plan error detected for token lookup, " - "retrying with fresh plan. This may occur during rolling deployments " - "when schema changes are applied." - ) - return await self.db.query_first(sql_query_retry, *args) - else: + if "cached plan must not change result type" not in str(e): raise + verbose_proxy_logger.warning( + "PostgreSQL cached plan error detected for token lookup; " + "recreating the database connection and retrying with the same " + "query. This may occur during rolling deployments when schema " + "changes are applied." + ) + await self.attempt_db_reconnect(reason="postgres_cached_plan_error") + return await self.db.query_first(sql_query, *args) @backoff.on_exception( backoff.expo, @@ -3548,7 +3557,10 @@ class PrismaClient: db=self.db, hashed_token=hashed_token ) if active_token_id: - response = await self.get_data( + # The recursive call returns a finished + # LiteLLM_VerificationTokenView; the dict + # normalization below would crash subscripting it. + deprecated_response = await self.get_data( token=active_token_id, table_name="combined_view", query_type="find_unique", @@ -3556,10 +3568,11 @@ class PrismaClient: proxy_logging_obj=proxy_logging_obj, check_deprecated=False, ) - if response is not None: + if deprecated_response is not None: verbose_proxy_logger.debug( "Deprecated key used during grace period" ) + return deprecated_response if response is not None: if response["team_models"] is None: diff --git a/pyproject.toml b/pyproject.toml index c7c28583808..bd7e002b717 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,7 +125,7 @@ proxy-runtime = [ "mangum>=0.17.0,<1.0", "azure-ai-contentsafety>=1.0.0,<2.0", "azure-storage-file-datalake>=12.20.0,<13.0", - "pypdf>=6.10.2,<7.0; python_version < '3.14'", + "pypdf>=6.12.0,<7.0; python_version < '3.14'", "llm-sandbox>=0.3.39,<1.0", "detect-secrets>=1.5.0,<2.0", ] @@ -231,6 +231,10 @@ requires = ["uv_build==0.11.8"] build-backend = "uv_build" [tool.uv] +constraint-dependencies = [ + "tornado>=6.5.6", + "aiohttp>=3.13.5,<3.14", +] default-groups = ["dev"] required-version = ">=0.10.9" exclude-newer = "3 days" diff --git a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py index 4ccde85dae2..fae315fb474 100644 --- a/tests/test_litellm/proxy/auth/test_auth_exception_handler.py +++ b/tests/test_litellm/proxy/auth/test_auth_exception_handler.py @@ -112,6 +112,157 @@ async def test_handle_authentication_error_data_layer_errors_do_not_fall_back( ) +@pytest.mark.asyncio +@pytest.mark.parametrize( + "db_error", + [ + ConnectionError("connection refused"), + TimeoutError("timed out"), + asyncio.TimeoutError(), + OSError("network is unreachable"), + HTTPClientClosedError(), + PrismaError("can't reach database server"), + RawQueryError( + data={ + "user_facing_error": { + "message": "cached plan must not change result type", + "meta": {"table": "t"}, + } + } + ), + ], +) +async def test_handle_authentication_error_db_infra_error_returns_503(db_error): + """Regression for the outage where valid keys got 401 for 4 hours: an + infrastructure-level DB failure during auth must surface as 503 (the DB + could not confirm the key), never as 401 ("Invalid API key").""" + handler = UserAPIKeyAuthExceptionHandler() + + with ( + patch( + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", + new_callable=AsyncMock, + return_value=None, + ), + patch( + "litellm.proxy.proxy_server.general_settings", + {"allow_requests_on_db_unavailable": False}, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error( + db_error, + MagicMock(), + {}, + "/v1/chat/completions", + None, + "sk-valid-but-db-down", + ) + + assert int(exc_info.value.code) == status.HTTP_503_SERVICE_UNAVAILABLE + assert exc_info.value.type == ProxyErrorTypes.no_db_connection + assert "Invalid API key" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +async def test_handle_authentication_error_prisma_engine_teardown_returns_503(): + """Regression for the first-request-of-an-outage edge case: at the instant + the DB socket drops, the prisma query engine returns a malformed error + payload and prisma-client-py crashes with a bare + ``AttributeError: 'NoneType' object has no attribute 'get'`` before it can + raise P1001. That AttributeError reached auth and fell through to 401. It + must surface as 503 like every other infra failure during the outage.""" + from prisma.engine import utils as prisma_engine_utils + + malformed_payload = [ + { + "error": "Can't reach database server", + "user_facing_error": { + "error_code": "P1001", + "message": "Can't reach database server at `localhost`:`5503`", + "meta": None, + }, + } + ] + try: + prisma_engine_utils.handle_response_errors(None, malformed_payload) + raise AssertionError("expected prisma to raise AttributeError") + except AttributeError as e: + teardown_error = e + + handler = UserAPIKeyAuthExceptionHandler() + + with ( + patch( + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", + new_callable=AsyncMock, + return_value=None, + ), + patch( + "litellm.proxy.proxy_server.general_settings", + {"allow_requests_on_db_unavailable": False}, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error( + teardown_error, + MagicMock(), + {}, + "/v1/chat/completions", + None, + "sk-valid-but-db-down", + ) + + assert int(exc_info.value.code) == status.HTTP_503_SERVICE_UNAVAILABLE + assert exc_info.value.type == ProxyErrorTypes.no_db_connection + assert "Invalid API key" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "auth_error", + [ + # DB returned no row -> get_key_object raises this exact 401. + ProxyException( + message="Authentication Error, Invalid proxy server token passed.", + type=ProxyErrorTypes.token_not_found_in_db, + param="key", + code=status.HTTP_401_UNAUTHORIZED, + ), + # A bare auth failure raised as a plain Exception (e.g. master-key-only + # route) must keep returning 401, not get reclassified as 503. + Exception("Invalid proxy server token passed"), + ], +) +async def test_handle_authentication_error_genuine_auth_failure_stays_401(auth_error): + """Guard against the 503 conversion being too broad: a genuine auth + failure (missing key / wrong key) must still be 401.""" + handler = UserAPIKeyAuthExceptionHandler() + + with ( + patch( + "litellm.proxy.proxy_server.proxy_logging_obj.post_call_failure_hook", + new_callable=AsyncMock, + return_value=None, + ), + patch( + "litellm.proxy.proxy_server.general_settings", + {"allow_requests_on_db_unavailable": False}, + ), + ): + with pytest.raises(ProxyException) as exc_info: + await handler._handle_authentication_error( + auth_error, + MagicMock(), + {}, + "/v1/chat/completions", + None, + "sk-bad-key", + ) + + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED + + @pytest.mark.asyncio async def test_handle_authentication_error_budget_exceeded(): handler = UserAPIKeyAuthExceptionHandler() diff --git a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py index a3452ac8024..37f34f82afa 100644 --- a/tests/test_litellm/proxy/auth/test_user_api_key_auth.py +++ b/tests/test_litellm/proxy/auth/test_user_api_key_auth.py @@ -110,11 +110,71 @@ async def test_should_clear_stale_budget_reservation_when_budget_checks_skip(): user_api_key_cache=MagicMock(), proxy_logging_obj=MagicMock(), skip_budget_checks=True, + general_settings={}, ) assert user_api_key_auth_obj.budget_reservation is None +@pytest.mark.asyncio +async def test_disable_budget_reservation_skips_reservation(): + """#27639: general_settings.disable_budget_reservation turns off the optimistic Redis + reservation so operators hit by phantom BudgetExceededError can opt out of it.""" + user_api_key_auth_obj = UserAPIKeyAuth(token="test_token") + + with patch( + "litellm.proxy.spend_tracking.budget_reservation.reserve_budget_for_request", + new=AsyncMock(return_value={"reserved_cost": 0.5, "entries": []}), + ) as mock_reserve: + await _reserve_budget_after_common_checks( + user_api_key_auth_obj=user_api_key_auth_obj, + request_data={"model": "gpt-4o"}, + route="/v1/chat/completions", + llm_router=None, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + skip_budget_checks=False, + general_settings={"disable_budget_reservation": True}, + ) + + mock_reserve.assert_not_called() + assert user_api_key_auth_obj.budget_reservation is None + + +@pytest.mark.asyncio +async def test_budget_reservation_runs_when_not_disabled(): + """Control for #27639: with the flag absent, the reservation still runs and is stored.""" + user_api_key_auth_obj = UserAPIKeyAuth(token="test_token") + reservation = { + "reserved_cost": 0.5, + "entries": [{"counter_key": "spend:key:test_token"}], + } + + with patch( + "litellm.proxy.spend_tracking.budget_reservation.reserve_budget_for_request", + new=AsyncMock(return_value=reservation), + ) as mock_reserve: + await _reserve_budget_after_common_checks( + user_api_key_auth_obj=user_api_key_auth_obj, + request_data={"model": "gpt-4o"}, + route="/v1/chat/completions", + llm_router=None, + team_object=None, + user_object=None, + prisma_client=None, + user_api_key_cache=MagicMock(), + proxy_logging_obj=MagicMock(), + skip_budget_checks=False, + general_settings={}, + ) + + mock_reserve.assert_awaited_once() + assert user_api_key_auth_obj.budget_reservation == reservation + + @pytest.mark.asyncio async def test_should_not_reuse_cached_key_object_for_request_state(): key_cache = DualCache() @@ -3459,3 +3519,114 @@ async def test_user_api_key_auth_does_not_overwrite_end_user_id_set_by_builder() finally: for k, v in originals.items(): setattr(_proxy_server_mod, k, v) + + +def _proxy_attrs_for_db_lookup(): + """Minimal proxy_server attributes for driving the real + ``_user_api_key_auth_builder`` down to the DB key lookup.""" + proxy_logging_obj = MagicMock() + proxy_logging_obj.post_call_failure_hook = AsyncMock(return_value=None) + return { + "prisma_client": MagicMock(), + "user_api_key_cache": DualCache(), + "proxy_logging_obj": proxy_logging_obj, + "master_key": "sk-test-master", + "general_settings": {"allow_requests_on_db_unavailable": False}, + "llm_model_list": [], + "llm_router": None, + "open_telemetry_logger": None, + "model_max_budget_limiter": MagicMock(), + "user_custom_auth": None, + "jwt_handler": None, + "litellm_proxy_admin_name": "admin", + } + + +async def _run_builder_with_key_lookup(get_key_object_mock): + """Drive the real auth builder with ``get_key_object`` replaced by the + given mock. Returns the builder result.""" + from fastapi import Request + from starlette.datastructures import URL + + import litellm.proxy.proxy_server as _proxy_server_mod + from litellm.proxy.auth.user_api_key_auth import _user_api_key_auth_builder + + attrs = _proxy_attrs_for_db_lookup() + originals = {a: getattr(_proxy_server_mod, a, None) for a in attrs} + try: + for k, v in attrs.items(): + setattr(_proxy_server_mod, k, v) + request = Request(scope={"type": "http"}) + request._url = URL(url="/chat/completions") + with ( + patch( + "litellm.proxy.auth.user_api_key_auth.get_key_object", + get_key_object_mock, + ), + ): + return await _user_api_key_auth_builder( + request=request, + api_key="Bearer sk-db-lookup-test", + azure_api_key_header="", + anthropic_api_key_header=None, + google_ai_studio_api_key_header=None, + azure_apim_header=None, + request_data={}, + ) + finally: + for k, v in originals.items(): + setattr(_proxy_server_mod, k, v) + + +@pytest.mark.asyncio +async def test_builder_returns_503_when_db_lookup_raises_infra_error(): + """End-to-end: a DB infrastructure failure during the key lookup must + propagate past the ``except ProxyException`` guard and surface as 503, + not the 401 that masked the 4-hour outage. Killing the new 503 branch + flips this to 401 and fails the test.""" + get_key_object = AsyncMock(side_effect=ConnectionError("connection refused")) + + with pytest.raises(ProxyException) as exc_info: + await _run_builder_with_key_lookup(get_key_object) + + assert int(exc_info.value.code) == status.HTTP_503_SERVICE_UNAVAILABLE + assert exc_info.value.type == ProxyErrorTypes.no_db_connection + assert "Invalid API key" not in str(exc_info.value.message) + + +@pytest.mark.asyncio +async def test_builder_returns_401_when_db_lookup_reports_missing_key(): + """Regression guard: a genuinely missing key (DB returned no row, which + ``get_key_object`` raises as a 401 ProxyException) must still be 401.""" + missing_key_error = ProxyException( + message="Authentication Error, Invalid proxy server token passed. key=..., not found in db.", + type=ProxyErrorTypes.token_not_found_in_db, + param="key", + code=status.HTTP_401_UNAUTHORIZED, + ) + get_key_object = AsyncMock(side_effect=missing_key_error) + + with pytest.raises(ProxyException) as exc_info: + await _run_builder_with_key_lookup(get_key_object) + + assert int(exc_info.value.code) == status.HTTP_401_UNAUTHORIZED + + +@pytest.mark.asyncio +async def test_builder_succeeds_when_db_lookup_returns_valid_token(): + """Regression guard: a valid key still authenticates. Proves the 503 + conversion only fires on the failure path and never intercepts success.""" + valid_token = UserAPIKeyAuth(api_key="sk-db-lookup-test", token="hashed-valid") + get_key_object = AsyncMock(return_value=valid_token) + + with patch( + "litellm.proxy.auth.user_api_key_auth._return_user_api_key_auth_obj", + new_callable=AsyncMock, + return_value=valid_token, + ) as mock_return: + result = await _run_builder_with_key_lookup(get_key_object) + + assert isinstance(result, UserAPIKeyAuth) + # Reaching the success-assembly return (never the exception handler) + # proves a valid key is unaffected by the 503 conversion. + mock_return.assert_awaited_once() diff --git a/tests/test_litellm/proxy/db/test_exception_handler.py b/tests/test_litellm/proxy/db/test_exception_handler.py index 9dcf5df4aeb..6021c221426 100644 --- a/tests/test_litellm/proxy/db/test_exception_handler.py +++ b/tests/test_litellm/proxy/db/test_exception_handler.py @@ -107,6 +107,201 @@ def test_is_database_connection_generic_errors(): ) +@pytest.mark.parametrize( + "error", + [ + ConnectionError("connection refused"), + TimeoutError("timed out"), + OSError("network is unreachable"), + asyncio.TimeoutError(), + HTTPClientClosedError(), + ClientNotConnectedError(), + PrismaError("can't reach database server"), + PrismaError(), + ], +) +def test_is_database_service_unavailable_error_infra_failures(error): + """Infrastructure-level failures (socket/connection/timeout, prisma + transport, unknown PrismaError) mean the DB could not answer, so auth + must surface 503 instead of treating a valid key as invalid.""" + assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is True + + +def test_is_database_service_unavailable_error_prisma_p1001_masquerades_as_dataerror(): + """Real-world regression: prisma-client-py raises the P1001 "can't reach + database server" connectivity failure as a DataError (a data-layer type). + A type-only check would miss it and return 401 during a genuine outage; + the message keyword must still classify it as service-unavailable -> 503.""" + p1001_as_dataerror = DataError( + data={ + "user_facing_error": { + "message": "Can't reach database server at `127.0.0.1`:`5499`", + "meta": {"table": "t"}, + } + } + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + p1001_as_dataerror + ) + is True + ) + + +def test_is_database_service_unavailable_error_cached_plan_escapes_as_503(): + """Composes with the cached-plan retry: when that recovery fails and the + Postgres "cached plan must not change result type" error escapes (raised by + prisma as a data-layer RawQueryError), it is a transient stale-DB-state + condition, not an invalid key, so it must classify as service-unavailable + -> 503 rather than fall through to 401.""" + cached_plan_error = RawQueryError( + data={ + "user_facing_error": { + "message": "cached plan must not change result type", + "meta": {"table": "t"}, + } + } + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + cached_plan_error + ) + is True + ) + + +def test_is_database_service_unavailable_error_prisma_engine_malformed_payload(): + """Real-world regression: at the instant the DB socket drops, the prisma + query engine returns a malformed error payload (``user_facing_error.meta`` + is ``null``). prisma-client-py's ``handle_response_errors`` then crashes + with ``AttributeError: 'NoneType' object has no attribute 'get'`` before it + can raise the proper P1001 error. That bare AttributeError has no + connection keyword, so without the prisma-engine-origin check it falls + through to 401 on the first request of an outage. Reproduce the exact + prisma crash and assert it classifies as service-unavailable -> 503.""" + from prisma.engine import utils as prisma_engine_utils + + malformed_payload = [ + { + "error": "Can't reach database server", + "user_facing_error": { + "error_code": "P1001", + "message": "Can't reach database server at `localhost`:`5503`", + "meta": None, + }, + } + ] + with pytest.raises(AttributeError) as exc_info: + prisma_engine_utils.handle_response_errors(None, malformed_payload) + + assert "no attribute 'get'" in str(exc_info.value) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) + is True + ) + + +def test_is_prisma_engine_internal_error_excludes_application_attributeerror(): + """The prisma-engine-origin check must stay narrow: a genuine AttributeError + raised by application code (a real bug) must NOT be classified as + service-unavailable, otherwise real bugs would silently become 503s.""" + + def application_bug(): + none_value = None + return none_value.get("oops") + + with pytest.raises(AttributeError) as exc_info: + application_bug() + + assert ( + PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc_info.value) + is False + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) + is False + ) + + +def test_is_prisma_engine_internal_error_excludes_data_layer_prisma_error(): + """A data-layer ``PrismaError`` (the DB IS reachable and rejected the data) + must stay 401. These are always raised from prisma internals, so the check + excludes any ``PrismaError`` by type before inspecting the traceback.""" + data_layer_error = UniqueViolationError( + data={"user_facing_error": {"meta": {"table": "t"}}} + ) + try: + raise data_layer_error + except UniqueViolationError as e: + assert PrismaDBExceptionHandler.is_prisma_engine_internal_error(e) is False + + +@pytest.mark.parametrize( + "error", + [ + DataError(data={"user_facing_error": {"meta": {"table": "t"}}}), + UniqueViolationError(data={"user_facing_error": {"meta": {"table": "t"}}}), + RecordNotFoundError(data={"user_facing_error": {"meta": {"table": "t"}}}), + Exception("some unrelated error"), + ValueError("bad value"), + ], +) +def test_is_database_service_unavailable_error_excludes_non_infra(error): + """Data-layer errors (the DB IS reachable and answered) and generic + non-DB errors must NOT be classified as service-unavailable, otherwise a + genuine 401 would be masked as a transient 503.""" + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is False + ) + + +def test_is_database_service_unavailable_error_asyncpg(monkeypatch): + """asyncpg connection/interface errors map to service-unavailable. asyncpg + is not a hard dependency, so inject a stand-in module to exercise the + branch deterministically regardless of the install environment.""" + import sys + import types + + fake_asyncpg = types.ModuleType("asyncpg") + fake_exceptions = types.ModuleType("asyncpg.exceptions") + + class PostgresConnectionError(Exception): + pass + + class InterfaceError(Exception): + pass + + class UniqueViolationError(Exception): # data-layer, must stay False + pass + + fake_exceptions.PostgresConnectionError = PostgresConnectionError + fake_exceptions.InterfaceError = InterfaceError + fake_exceptions.UniqueViolationError = UniqueViolationError + fake_asyncpg.exceptions = fake_exceptions + + monkeypatch.setitem(sys.modules, "asyncpg", fake_asyncpg) + monkeypatch.setitem(sys.modules, "asyncpg.exceptions", fake_exceptions) + + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + PostgresConnectionError("connection reset") + ) + is True + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + InterfaceError("connection was closed") + ) + is True + ) + assert ( + PrismaDBExceptionHandler.is_database_service_unavailable_error( + UniqueViolationError("duplicate key") + ) + is False + ) + + # Test should_allow_request_on_db_unavailable method @patch( "litellm.proxy.proxy_server.general_settings", diff --git a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py index 85c7c130b36..f16074a049b 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_model_management_endpoints.py @@ -1129,6 +1129,307 @@ class TestTeamModelUpdate: ) assert "403" in str(exc_info.value) + def test_get_public_model_name_28382_dashboard_echo_preserves_public_name(self): + """Regression for #28382 - a non-rename dashboard PATCH echoes the + internal generated model_name (model_name_{team}_{uuid}) at the top + level. That internal-shape value must be ignored (not treated as a + rename), so _get_public_model_name falls through to the existing public + name instead of overwriting it with the internal one.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_name="model_name_test-team_abc123", + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + def test_get_public_model_name_preserves_db_public_name_when_internal_name_unchanged( + self, + ): + """If patch_data.model_info has no team_public_model_name and + patch_data.model_name equals db_model.model_name (dashboard re-sending + the internal name without touching the public-name field), the + existing db_model.model_info.team_public_model_name must be preserved.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_name="model_name_test-team_abc123", + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + def test_get_public_model_name_allows_top_level_rename(self): + """A genuine rename via the top-level model_name field (no + patch_data.model_info.team_public_model_name supplied, and the new + name differs from the existing internal db model_name) must still + return the new name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="old-public-name", + ), + ) + patch_data = updateDeployment( + model_name="new-public-name", + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "new-public-name" + ) + + def test_get_public_model_name_top_level_rename_wins_over_stale_model_info(self): + """Regression (codex review): on a dashboard rename the UI sends the new + name in model_name but passes the existing model_info blob through + untouched -- so it still carries the OLD team_public_model_name. The + top-level rename must win; otherwise _update_existing_team_model_assignment + sees no change, never updates the team ACL, and the rename is silently + dropped while the UI optimistically shows the new name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_team-a_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-4.1"), + model_info=ModelInfo( + team_id="team-a", team_public_model_name="old-public-name" + ), + ) + patch_data = updateDeployment( + model_name="new-public-name", + model_info=ModelInfo( + team_id="team-a", + team_public_model_name="old-public-name", # stale, untouched by UI + ), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "new-public-name" + ) + + def test_get_public_model_name_falls_back_to_db_public_name(self): + """When patch_data carries no name hints at all (neither model_name + nor model_info.team_public_model_name), fall back to the existing + db_model.model_info.team_public_model_name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + def test_get_public_model_name_last_resort_returns_db_model_name(self): + """Legacy rows may have no team_public_model_name anywhere; the + function must still return a string (the existing db_model.model_name) + rather than raising.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="legacy-model", + litellm_params=LiteLLM_Params(model="azure/legacy"), + model_info=ModelInfo(team_id="test-team"), + ) + patch_data = updateDeployment( + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "legacy-model" + ) + + def test_get_public_model_name_ignores_different_internal_shape_name(self): + """A stale client may PATCH an internal-shaped model_name that does not + equal the current DB column (e.g. a different uuid). It must NOT be + treated as a rename -- fall through to the existing public name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_realuuid", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_name="model_name_test-team_differentuuid", + model_info=ModelInfo(team_id="test-team"), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + def test_get_public_model_name_ignores_internal_shape_patch_public(self): + """If a corrupted row round-trips an internal-shaped value in + model_info.team_public_model_name, it must not be accepted as the + public name -- fall through to the existing db public name.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _get_public_model_name, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_realuuid", + litellm_params=LiteLLM_Params(model="azure/gpt-5.2-low-rpm-testing"), + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_info=ModelInfo( + team_id="test-team", + team_public_model_name="model_name_test-team_realuuid", + ), + ) + + assert ( + _get_public_model_name(patch_data=patch_data, db_model=db_model) + == "gpt-5.2-low-rpm-testing" + ) + + @pytest.mark.asyncio + async def test_dashboard_edit_preserves_public_name_and_acl(self): + """End-to-end regression for #28382: PATCH payload shaped like the + dashboard's model-edit form (top-level model_name = internal generated + name, model_info.team_public_model_name = public name) must NOT trigger + a public-name rename, must NOT touch the team ACL, and must serialize + the public name back into model_info.""" + from litellm.proxy.management_endpoints.model_management_endpoints import ( + _update_team_model_in_db, + ) + from litellm.types.router import ModelInfo + + db_model = Deployment( + model_name="model_name_test-team_abc123", + litellm_params=LiteLLM_Params( + model="azure/gpt-5.2-low-rpm-testing", + custom_llm_provider="azure", + ), + model_info=ModelInfo( + id="model-id-123", + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + patch_data = updateDeployment( + model_name="model_name_test-team_abc123", + litellm_params=None, + model_info=ModelInfo( + id="model-id-123", + team_id="test-team", + team_public_model_name="gpt-5.2-low-rpm-testing", + ), + ) + user_api_key_dict = UserAPIKeyAuth( + user_id="test_user", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + prisma_client = MockPrismaClient(team_exists=True) + + with ( + patch( + "litellm.proxy.proxy_server.premium_user", + True, + ), + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_add" + ) as mock_team_model_add, + patch( + "litellm.proxy.management_endpoints.model_management_endpoints.team_model_delete" + ) as mock_team_model_delete, + ): + result = await _update_team_model_in_db( + db_model=db_model, + patch_data=patch_data, + user_api_key_dict=user_api_key_dict, + prisma_client=prisma_client, # type: ignore + ) + + # team ACL must not be touched on a no-op edit + mock_team_model_add.assert_not_called() + mock_team_model_delete.assert_not_called() + + # the merged model_info written to the DB must keep the public name + model_info_json = result.get("model_info", "") + parsed_model_info = json.loads(model_info_json) + assert ( + parsed_model_info.get("team_public_model_name") == "gpt-5.2-low-rpm-testing" + ) + + # the internal model_name must not have been overwritten (caller + # intentionally clears patch_data.model_name so the DB row's name + # column is left alone) + assert result.get("model_name") == "model_name_test-team_abc123" + class TestModelInfoEndpoint: """Test the model_info endpoint for retrieving individual model information""" diff --git a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py index f8b6fbde3dc..78816f9ff2d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_anthropic_passthrough_logging_handler.py @@ -321,6 +321,270 @@ class TestAzureAnthropicCostCalculation: assert call_kwargs["model"] == "azure_ai/claude-sonnet-4-5_gb_20250929" assert call_kwargs["custom_llm_provider"] == "azure_ai" + @patch("litellm.completion_cost") + def test_cost_calculation_resolves_unknown_model_from_litellm_params( + self, mock_completion_cost + ): + """When the body model is the "unknown" sentinel, the deployment model + from litellm_params must be used for costing, not "unknown" (which makes + completion_cost raise and the cost silently fall back to $0).""" + from datetime import datetime + + from litellm.types.utils import ModelResponse + + mock_completion_cost.return_value = 0.001 + + logging_obj = self._create_mock_logging_obj(model="unknown") + logging_obj.model_call_details["litellm_params"] = { + "model": "anthropic/claude-3-5-haiku-20241022", + "metadata": { + "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" + }, + } + logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] + + mock_response = MagicMock(spec=ModelResponse) + mock_response.id = "test-id" + mock_response.model = "unknown" + + kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=mock_response, + model="unknown", + kwargs={}, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + ) + + mock_completion_cost.assert_called_once() + assert ( + mock_completion_cost.call_args[1]["model"] + == "anthropic/claude-3-5-haiku-20241022" + ) + assert kwargs["response_cost"] == 0.001 + assert kwargs["model"] == "anthropic/claude-3-5-haiku-20241022" + + @patch("litellm.completion_cost") + def test_cost_calculation_resolves_unknown_model_from_model_group( + self, mock_completion_cost + ): + """With only model_group available (no deployment litellm_params.model), + the leading passthrough/ prefix must be stripped so the cost map can + resolve the model.""" + from datetime import datetime + + from litellm.types.utils import ModelResponse + + mock_completion_cost.return_value = 0.002 + + logging_obj = self._create_mock_logging_obj(model="unknown") + logging_obj.model_call_details["litellm_params"] = { + "metadata": { + "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" + } + } + logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] + + mock_response = MagicMock(spec=ModelResponse) + mock_response.id = "test-id" + mock_response.model = "unknown" + + kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=mock_response, + model="unknown", + kwargs={}, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + ) + + mock_completion_cost.assert_called_once() + assert ( + mock_completion_cost.call_args[1]["model"] + == "anthropic/claude-3-5-haiku-20241022" + ) + assert kwargs["response_cost"] == 0.002 + + @patch("litellm.completion_cost") + def test_cost_calculation_skips_unknown_litellm_params_model_for_model_group( + self, mock_completion_cost + ): + """When litellm_params.model is itself the "unknown" sentinel, the + deployment-model branch must not short-circuit; resolution falls through + to model_group so costing still prices the real model instead of "unknown".""" + from datetime import datetime + + from litellm.types.utils import ModelResponse + + mock_completion_cost.return_value = 0.003 + + logging_obj = self._create_mock_logging_obj(model="unknown") + logging_obj.model_call_details["litellm_params"] = { + "model": "unknown", + "metadata": { + "model_group": "passthrough/anthropic/claude-3-5-haiku-20241022" + }, + } + logging_obj.litellm_params = logging_obj.model_call_details["litellm_params"] + + mock_response = MagicMock(spec=ModelResponse) + mock_response.id = "test-id" + mock_response.model = "unknown" + + kwargs = AnthropicPassthroughLoggingHandler._create_anthropic_response_logging_payload( + litellm_model_response=mock_response, + model="unknown", + kwargs={}, + start_time=datetime.now(), + end_time=datetime.now(), + logging_obj=logging_obj, + ) + + mock_completion_cost.assert_called_once() + assert ( + mock_completion_cost.call_args[1]["model"] + == "anthropic/claude-3-5-haiku-20241022" + ) + assert kwargs["response_cost"] == 0.003 + assert kwargs["model"] == "anthropic/claude-3-5-haiku-20241022" + + @patch("litellm.completion_cost") + def test_streaming_cost_calculation_resolves_model_from_message_start_chunk( + self, mock_completion_cost + ): + """On the bare /anthropic passthrough path litellm_params carries no model + or model_group and the body model is the "unknown" sentinel; the model + must be recovered from the message_start SSE event so completion_cost + prices the real model instead of failing on "unknown" and logging $0.""" + from datetime import datetime + + from litellm.litellm_core_utils.litellm_logging import ( + Logging as RealLoggingObj, + ) + from litellm.proxy.pass_through_endpoints.streaming_handler import ( + PassThroughStreamingHandler, + ) + + mock_completion_cost.return_value = 0.001 + + def _sse(event, data): + return f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() + + frames = [ + _sse( + "message_start", + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-3-5-haiku-20241022", + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 0}, + }, + }, + ), + _sse( + "content_block_start", + { + "type": "content_block_start", + "index": 0, + "content_block": {"type": "text", "text": ""}, + }, + ), + _sse( + "content_block_delta", + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": "hi"}, + }, + ), + _sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _sse( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 1}, + }, + ), + _sse("message_stop", {"type": "message_stop"}), + ] + all_chunks = list( + PassThroughStreamingHandler._convert_raw_bytes_to_str_lines(frames) + ) + + logging_obj = RealLoggingObj( + model="unknown", + messages=[{"role": "user", "content": "hi"}], + stream=True, + call_type="pass_through_endpoint", + start_time=datetime.now(), + litellm_call_id="test-call-id", + function_id="1", + ) + logging_obj.model_call_details["model"] = "unknown" + logging_obj.model_call_details["stream"] = True + logging_obj.model_call_details["litellm_params"] = {} + logging_obj.litellm_params = {} + + result = AnthropicPassthroughLoggingHandler._handle_logging_anthropic_collected_chunks( + litellm_logging_obj=logging_obj, + passthrough_success_handler_obj=MagicMock(), + url_route="/anthropic/v1/messages", + request_body={"stream": True}, + endpoint_type="messages", + start_time=datetime.now(), + all_chunks=all_chunks, + end_time=datetime.now(), + ) + + assert result["result"] is not None + mock_completion_cost.assert_called_once() + assert mock_completion_cost.call_args[1]["model"] == "claude-3-5-haiku-20241022" + assert result["kwargs"]["response_cost"] == 0.001 + assert result["kwargs"]["model"] == "claude-3-5-haiku-20241022" + + def test_extract_model_skips_non_dict_data_payload(self): + """A scalar data: payload (e.g. `data: null`) must be skipped, not crash + the streaming log handler with AttributeError, which would propagate out + and break spend logging for the whole request.""" + chunks = [ + "event: ping\ndata: null\n\n", + 'event: message_start\ndata: {"type": "message_start", "message": ' + '{"model": "claude-3-5-haiku-20241022"}}\n\n', + ] + + assert ( + AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks( + chunks + ) + == "claude-3-5-haiku-20241022" + ) + + def test_extract_model_parses_per_line_not_first_data_substring(self): + """A raw multi-line SSE event whose non-data line contains the substring + "data:" must not derail parsing: matching only lines that start with + "data:" recovers the message_start model, whereas a first-substring slice + would consume the wrong offset, fail to parse JSON, and return None.""" + raw_event = ( + "event: ping data: not-json\n" + 'data: {"type": "message_start", "message": ' + '{"model": "claude-3-5-haiku-20241022"}}\n\n' + ) + + assert ( + AnthropicPassthroughLoggingHandler._extract_model_from_anthropic_chunks( + [raw_event] + ) + == "claude-3-5-haiku-20241022" + ) + + class TestAnthropicBatchPassthroughCostTracking: """Test cases for Anthropic batch passthrough cost tracking functionality""" @@ -1045,6 +1309,72 @@ class TestPureTextFastPathParity: ) +class TestBuildCompleteStreamingResponseRobustness: + """_build_complete_streaming_response must tolerate non-standard SSE frames.""" + + def _build(self, chunks: List[str]): + return AnthropicPassthroughLoggingHandler._build_complete_streaming_response( + all_chunks=chunks, + litellm_logging_obj=MagicMock(), + model="claude-3-sonnet-20240229", + ) + + def test_done_frame_is_skipped(self): + """A bare 'data: [DONE]' control frame must not break reconstruction.""" + chunks = [ + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-3-sonnet-20240229","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hi"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":2}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + "data: [DONE]", + ] + result = self._build(chunks) + assert result is not None + assert result.choices[0].message.content == "Hi" + + def test_non_json_sse_line_is_skipped(self): + """Non-JSON SSE lines (comments, keep-alive pings) must be skipped.""" + chunks = [ + ": ping", + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-3-sonnet-20240229","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}', + "this is not json at all", + ] + # Must not raise; a malformed stream simply yields no usable response. + result = self._build(chunks) + assert result is None or hasattr(result, "choices") + + def test_mixed_valid_and_invalid_frames(self): + """Valid events are still collected when interleaved with invalid ones.""" + chunks = [ + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-3-sonnet-20240229","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}', + "data: [DONE]", + ": keep-alive", + "not-json", + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"Hello"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":2}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ] + result = self._build(chunks) + assert result is not None + assert result.choices[0].message.content == "Hello" + + def test_done_in_text_payload_is_not_dropped(self): + """A valid event whose text content contains '[DONE]' must NOT be skipped.""" + chunks = [ + 'event: message_start\ndata: {"type":"message_start","message":{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-3-sonnet-20240229","stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":10,"output_tokens":1}}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"The stream ends with [DONE]"}}', + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + 'event: message_delta\ndata: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},"usage":{"output_tokens":8}}', + 'event: message_stop\ndata: {"type":"message_stop"}', + ] + result = self._build(chunks) + assert result is not None + assert result.choices[0].message.content == "The stream ends with [DONE]" class TestStreamFalseDeduplication: """ Regression tests for the duplicate-callback bug where a streaming pass-through diff --git a/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py new file mode 100644 index 00000000000..6a8e0d15d8b --- /dev/null +++ b/tests/test_litellm/proxy/proxy_server/test_team_model_name_translation.py @@ -0,0 +1,595 @@ +"""Coverage for team-scoped model-name translation in /model/info responses. + +These live in tests/test_litellm/proxy/proxy_server/ (not the top-level +test_proxy_server.py) because the CI coverage job collects this directory. +They exercise the read-path fix for issue #28382: `/v1`, `/v2`, and +`/model/info` must surface `model_info.team_public_model_name` for team-scoped +rows instead of the internal routing key `model_name_{team_id}_{uuid}`. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import litellm.proxy.proxy_server as ps +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.proxy_server import ( + _get_proxy_model_info, + _translate_model_name_for_response, +) + + +def _team_row() -> dict: + return { + "model_name": "model_name_team-abc-123_4a6b8", + "litellm_params": {"model": "azure/gpt-5.2-low-rpm-testing"}, + "model_info": { + "id": "byok-id-1", + "team_id": "team-abc-123", + "team_public_model_name": "team-claude-sonnet", + "db_model": True, + }, + } + + +def test_translate_swaps_internal_name_for_public(): + """Team-scoped row: model_name is swapped to the public name.""" + result = _translate_model_name_for_response(_team_row()) + assert result["model_name"] == "team-claude-sonnet" + + +def test_translate_leaves_global_row_untouched(): + """No team_id / team_public_model_name -> pass through unchanged.""" + model = { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "normal-id-1", "db_model": False}, + } + assert _translate_model_name_for_response(model)["model_name"] == "gpt-4o" + + +def test_translate_leaves_non_internal_shape_untouched(): + """Team row whose model_name is not the internal routing key is not rewritten.""" + model = _team_row() + model["model_name"] = "already-public-name" + assert ( + _translate_model_name_for_response(model)["model_name"] == "already-public-name" + ) + + +def test_translate_handles_missing_or_non_dict_model_info(): + """Missing / None / non-dict model_info, and a non-dict model, must not raise.""" + # missing model_info + assert _translate_model_name_for_response({"model_name": "x"})["model_name"] == "x" + # model_info is None -> coerced to {} -> no team fields + assert ( + _translate_model_name_for_response({"model_name": "x", "model_info": None})[ + "model_name" + ] + == "x" + ) + # model_info is a truthy non-dict (e.g. a stray string) -> early return + assert ( + _translate_model_name_for_response( + {"model_name": "x", "model_info": "garbage"} + )["model_name"] + == "x" + ) + # model itself is not a dict + assert _translate_model_name_for_response("not-a-dict") == "not-a-dict" # type: ignore[arg-type] + + +def test_translate_does_not_mutate_input(): + """Returns a shallow copy; the router's in-memory list keeps the routing key.""" + model = _team_row() + result = _translate_model_name_for_response(model) + assert result is not model + assert model["model_name"] == "model_name_team-abc-123_4a6b8" + + +def test_get_proxy_model_info_returns_public_name_for_team_row(): + """`_get_proxy_model_info` must return the public name for a team-scoped + row. Because _translate_model_name_for_response returns a shallow copy + (it does not mutate), callers MUST use the return value -- the + `/v1/model/info` list path historically discarded it, leaking the internal + routing key (#28382).""" + # Mirror the (fixed) /v1/model/info list path: assign the return back. + all_models = [_get_proxy_model_info(model=m) for m in [_team_row()]] + assert all_models[0]["model_name"] == "team-claude-sonnet" + + +@pytest.mark.asyncio +async def test_model_info_v2_translates_team_model_name(monkeypatch): + """/v2/model/info must surface the public name for team-scoped rows. + Covers the translation step in model_info_v2 (the read-path call site).""" + router = MagicMock() + router.model_list = [_team_row()] + + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(ps.proxy_config, "get_config", AsyncMock(return_value={})) + monkeypatch.setattr( + ps, + "_apply_search_filter_to_models", + AsyncMock(side_effect=lambda all_models, **kw: (all_models, len(all_models))), + ) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + import litellm.proxy.agent_endpoints.model_list_helpers as mlh + + monkeypatch.setattr( + mlh, + "append_agents_to_model_info", + AsyncMock(side_effect=lambda models, **kw: models), + ) + + admin = UserAPIKeyAuth(user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN) + # Pass every query param explicitly: called directly (not through FastAPI), + # the fastapi.Query(...) defaults are Query objects, not their values. + resp = await ps.model_info_v2( + user_api_key_dict=admin, + model=None, + user_models_only=False, + include_team_models=False, + debug=False, + page=1, + size=50, + search=None, + modelId=None, + teamId=None, + sortBy=None, + sortOrder="asc", + ) + + names = [m["model_name"] for m in resp["data"]] + assert "team-claude-sonnet" in names + assert "model_name_team-abc-123_4a6b8" not in names + + +@pytest.mark.asyncio +async def test_model_info_v1_list_path_translates_team_model_name(monkeypatch): + """/v1/model/info list path (no litellm_model_id) must include team-scoped + deployments from the router model list and surface the public name (#28382).""" + team_row = _team_row() + global_row = { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "normal-id-1", "db_model": False}, + } + router = MagicMock() + router.model_list = [team_row, global_row] + router.get_model_names.return_value = ["gpt-4o"] + router.get_model_access_groups.return_value = {} + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + + admin = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN, team_models=[] + ) + resp = await ps.model_info_v1(user_api_key_dict=admin, litellm_model_id=None) + + names = [m["model_name"] for m in resp["data"]] + assert "team-claude-sonnet" in names + assert "model_name_team-abc-123_4a6b8" not in names + + +@pytest.mark.asyncio +async def test_model_info_v1_unrestricted_key_returns_all_deployments(monkeypatch): + """Unrestricted keys must see all router deployments (legacy v1 access logic).""" + deployment = { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + router = MagicMock() + router.model_list = [deployment] + router.get_model_names.return_value = ["gpt-4"] + router.get_model_access_groups.return_value = {} + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + + caller = UserAPIKeyAuth( + user_id="user-1", + user_role=LitellmUserRoles.INTERNAL_USER, + models=[], + team_models=[], + ) + resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) + + assert [m["model_name"] for m in resp["data"]] == ["gpt-4"] + + +@pytest.mark.asyncio +async def test_model_info_v1_restricted_key_filters_deployments(monkeypatch): + """Key-level model allowlists must filter router deployments.""" + team_row = _team_row() + global_row = { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + router = MagicMock() + router.model_list = [team_row, global_row] + router.get_model_names.return_value = ["gpt-4", "team-claude-sonnet"] + router.get_model_access_groups.return_value = {} + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + + caller = UserAPIKeyAuth( + user_id="user-1", + user_role=LitellmUserRoles.INTERNAL_USER, + models=["gpt-4"], + team_models=[], + ) + resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) + + assert [m["model_name"] for m in resp["data"]] == ["gpt-4"] + + +def _other_team_row() -> dict: + return { + "model_name": "model_name_team-other_9f2c1", + "litellm_params": { + "model": "azure/gpt-5.2-low-rpm-testing", + "api_base": "https://team-other-private.example.com", + }, + "model_info": { + "id": "byok-id-other", + "team_id": "team-other", + "team_public_model_name": "team-claude-sonnet", + "db_model": True, + }, + } + + +@pytest.mark.asyncio +async def test_model_info_v1_unrestricted_key_hides_other_team_byok(monkeypatch): + """Unrestricted non-admin keys must not enumerate other teams' BYOK + deployments, but must still see global models and their own team's.""" + team_row = _team_row() + other_team_row = _other_team_row() + global_row = { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + router = MagicMock() + router.model_list = [team_row, other_team_row, global_row] + router.get_model_names.return_value = ["gpt-4"] + router.get_model_access_groups.return_value = {} + + prisma_client = MagicMock() + caller_user_row = MagicMock() + caller_user_row.teams = ["team-abc-123"] + caller_user_row.model_dump.return_value = { + "user_id": "user-1", + "teams": ["team-abc-123"], + "models": [], + } + prisma_client.db.litellm_usertable.find_unique = AsyncMock( + return_value=caller_user_row + ) + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", prisma_client) + monkeypatch.setattr(ps, "get_all_team_models", AsyncMock(return_value={})) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + + caller = UserAPIKeyAuth( + user_id="user-1", + user_role=LitellmUserRoles.INTERNAL_USER, + models=[], + team_models=[], + ) + resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) + + returned_ids = {m["model_info"]["id"] for m in resp["data"]} + assert returned_ids == {"global-id-1", "byok-id-1"} + assert "byok-id-other" not in returned_ids + names = [m["model_name"] for m in resp["data"]] + assert "team-claude-sonnet" in names + assert "gpt-4" in names + + +@pytest.mark.asyncio +async def test_model_info_v1_service_key_hides_all_team_byok(monkeypatch): + """A key without a resolvable user (e.g. CI/service token) sees only + global deployments, never any team-scoped BYOK rows.""" + team_row = _team_row() + other_team_row = _other_team_row() + global_row = { + "model_name": "gpt-4", + "litellm_params": {"model": "gpt-4"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + router = MagicMock() + router.model_list = [team_row, other_team_row, global_row] + router.get_model_names.return_value = ["gpt-4"] + router.get_model_access_groups.return_value = {} + + prisma_client = MagicMock() + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", prisma_client) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + + caller = UserAPIKeyAuth( + user_id=None, + user_role=LitellmUserRoles.INTERNAL_USER, + team_id="team-abc-123", + models=[], + team_models=[], + ) + resp = await ps.model_info_v1(user_api_key_dict=caller, litellm_model_id=None) + + assert [m["model_info"]["id"] for m in resp["data"]] == ["global-id-1"] + + +@pytest.mark.asyncio +async def test_model_info_v1_populates_access_via_team_ids(monkeypatch): + """`/v1/model/info` must populate access_via_team_ids when the DB is connected.""" + team_id = "team-abc-123" + team_row = _team_row() + global_row = { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + router = MagicMock() + router.model_list = [team_row, global_row] + router.get_model_names.return_value = ["gpt-4o", "team-claude-sonnet"] + router.get_model_access_groups.return_value = {} + router.get_model_ids.return_value = ["global-id-1"] + + prisma_client = MagicMock() + + async def _fake_populate(**kwargs): + for model in kwargs["all_models"]: + model_id = model["model_info"]["id"] + if model_id == "byok-id-1": + model["model_info"]["access_via_team_ids"] = [team_id] + model["model_info"]["direct_access"] = False + elif model_id == "global-id-1": + model["model_info"]["direct_access"] = True + return kwargs["all_models"] + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", prisma_client) + monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) + monkeypatch.setattr( + ps, "_enrich_model_info_with_litellm_data", lambda model, **kw: model + ) + + admin = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN, team_models=[] + ) + resp = await ps.model_info_v1(user_api_key_dict=admin, litellm_model_id=None) + + by_id = {m["model_info"]["id"]: m for m in resp["data"]} + assert by_id["byok-id-1"]["model_info"]["access_via_team_ids"] == [team_id] + assert by_id["byok-id-1"]["model_info"]["direct_access"] is False + assert by_id["global-id-1"]["model_info"]["direct_access"] is True + + +@pytest.mark.asyncio +async def test_populate_team_access_sets_direct_access_false_by_default(monkeypatch): + """Team-accessible models without direct access must return direct_access=false.""" + team_row = _team_row() + global_row = { + "model_name": "gpt-4o", + "litellm_params": {"model": "gpt-4o"}, + "model_info": {"id": "global-id-1", "db_model": False}, + } + router = MagicMock() + router.get_model_ids.return_value = ["global-id-1"] + monkeypatch.setattr( + ps, + "get_all_team_models", + AsyncMock(return_value={"byok-id-1": ["team-abc-123"]}), + ) + + admin = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN, team_models=[] + ) + result = await ps._populate_team_access_on_models( + user_api_key_dict=admin, + prisma_client=MagicMock(), + llm_router=router, + all_models=[team_row, global_row], + ) + + by_id = {m["model_info"]["id"]: m for m in result} + assert by_id["byok-id-1"]["model_info"]["direct_access"] is False + assert by_id["global-id-1"]["model_info"]["direct_access"] is True + + +@pytest.mark.asyncio +async def test_model_info_v1_team_id_without_db_fails_fast(monkeypatch): + """`teamId` without a connected DB raises 500 before any enrichment work runs.""" + router = MagicMock() + router.model_list = [_team_row()] + + enrich_spy = MagicMock(side_effect=lambda model, **kw: model) + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", None) + monkeypatch.setattr(ps, "_enrich_model_info_with_litellm_data", enrich_spy) + + admin = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN, team_models=[] + ) + + with pytest.raises(ps.HTTPException) as exc_info: + await ps.model_info_v1( + user_api_key_dict=admin, litellm_model_id=None, teamId="team-abc-123" + ) + + assert exc_info.value.status_code == 500 + assert "DB not connected" in exc_info.value.detail["error"] + enrich_spy.assert_not_called() + + +@pytest.mark.asyncio +async def test_model_info_v1_include_team_models_without_db_fails_fast(monkeypatch): + """`include_team_models` without a connected DB raises 500 instead of silently + returning an empty list (the access fields can only be populated from the DB).""" + router = MagicMock() + router.model_list = [_team_row()] + + enrich_spy = MagicMock(side_effect=lambda model, **kw: model) + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", None) + monkeypatch.setattr(ps, "_enrich_model_info_with_litellm_data", enrich_spy) + + admin = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN, team_models=[] + ) + + with pytest.raises(ps.HTTPException) as exc_info: + await ps.model_info_v1( + user_api_key_dict=admin, litellm_model_id=None, include_team_models=True + ) + + assert exc_info.value.status_code == 500 + assert "DB not connected" in exc_info.value.detail["error"] + enrich_spy.assert_not_called() + + +@pytest.mark.asyncio +async def test_model_info_v1_litellm_model_id_team_id_without_db_fails_fast( + monkeypatch, +): + """`litellm_model_id` + `teamId` without a connected DB must raise 500 too, not + return 200 with a model dict missing direct_access/access_via_team_ids.""" + router = MagicMock() + router.model_list = [_team_row()] + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", router.model_list) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", None) + + admin = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN, team_models=[] + ) + + with pytest.raises(ps.HTTPException) as exc_info: + await ps.model_info_v1( + user_api_key_dict=admin, + litellm_model_id="byok-id-1", + teamId="team-abc-123", + ) + + assert exc_info.value.status_code == 500 + assert "DB not connected" in exc_info.value.detail["error"] + router.get_deployment.assert_not_called() + + +@pytest.mark.asyncio +async def test_model_info_v1_litellm_model_id_include_team_models_filters_inaccessible( + monkeypatch, +): + """`litellm_model_id` + `include_team_models` must drop a model the caller cannot + use instead of returning it unconditionally from the single-model lookup.""" + team_row = _team_row() + + router = MagicMock() + deployment = MagicMock() + deployment.model_dump.return_value = team_row + router.get_deployment.return_value = deployment + + async def _fake_populate(**kwargs): + for model in kwargs["all_models"]: + model["model_info"]["direct_access"] = False + model["model_info"]["access_via_team_ids"] = [] + return kwargs["all_models"] + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", [team_row]) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row) + monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) + + caller = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.INTERNAL_USER, team_models=[] + ) + resp = await ps.model_info_v1( + user_api_key_dict=caller, + litellm_model_id="byok-id-1", + include_team_models=True, + ) + + assert resp["data"] == [] + + +@pytest.mark.asyncio +async def test_model_info_v1_litellm_model_id_team_id_applies_team_filter(monkeypatch): + """`litellm_model_id` + `teamId` must run the teamId filter on the single model + rather than returning it regardless of the team's access.""" + team_row = _team_row() + + router = MagicMock() + deployment = MagicMock() + deployment.model_dump.return_value = team_row + router.get_deployment.return_value = deployment + + async def _fake_populate(**kwargs): + return kwargs["all_models"] + + team_filter = AsyncMock(return_value=[]) + + monkeypatch.setattr(ps, "user_model", None) + monkeypatch.setattr(ps, "llm_model_list", [team_row]) + monkeypatch.setattr(ps, "llm_router", router) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + monkeypatch.setattr(ps, "_get_proxy_model_info", lambda model: team_row) + monkeypatch.setattr(ps, "_populate_team_access_on_models", _fake_populate) + monkeypatch.setattr(ps, "_filter_models_by_team_id", team_filter) + + admin = UserAPIKeyAuth( + user_id="u", user_role=LitellmUserRoles.PROXY_ADMIN, team_models=[] + ) + resp = await ps.model_info_v1( + user_api_key_dict=admin, + litellm_model_id="byok-id-1", + teamId="other-team", + ) + + assert resp["data"] == [] + team_filter.assert_awaited_once() + assert team_filter.await_args.kwargs["team_id"] == "other-team" + assert team_filter.await_args.kwargs["all_models"] == [team_row] diff --git a/tests/test_litellm/proxy/test_model_info_default_limits.py b/tests/test_litellm/proxy/test_model_info_default_limits.py index 641199c96f0..8111a7af006 100644 --- a/tests/test_litellm/proxy/test_model_info_default_limits.py +++ b/tests/test_litellm/proxy/test_model_info_default_limits.py @@ -146,9 +146,9 @@ class TestModelInfoEndpointWithRouter: deployment_dict = deployment.model_dump(exclude_none=True) mock_router = MagicMock() + mock_router.model_list = [deployment_dict] mock_router.get_model_names.return_value = ["model1"] mock_router.get_model_access_groups.return_value = {} - mock_router.get_model_list.return_value = [deployment_dict] user_api_key_dict = UserAPIKeyAuth(api_key="sk-test") @@ -156,6 +156,7 @@ class TestModelInfoEndpointWithRouter: patch("litellm.proxy.proxy_server.llm_router", mock_router), patch("litellm.proxy.proxy_server.llm_model_list", [deployment_dict]), patch("litellm.proxy.proxy_server.user_model", None), + patch("litellm.proxy.proxy_server.prisma_client", None), patch("litellm.proxy.proxy_server.get_key_models", return_value=["model1"]), patch( "litellm.proxy.proxy_server.get_team_models", return_value=["model1"] diff --git a/tests/test_litellm/proxy/test_proxy_cli.py b/tests/test_litellm/proxy/test_proxy_cli.py index 580ed95062b..3b3b4387bc7 100644 --- a/tests/test_litellm/proxy/test_proxy_cli.py +++ b/tests/test_litellm/proxy/test_proxy_cli.py @@ -707,6 +707,127 @@ class TestProxyInitializationHelpers: assert appended_params["pgbouncer"] == "true" assert appended_params["statement_cache_size"] == 0 + def test_build_db_connection_url_params_disable_prepared_statements(self): + from litellm.proxy.proxy_cli import _build_db_connection_url_params + + params = _build_db_connection_url_params( + connection_limit=10, + pool_timeout=60, + disable_prepared_statements=True, + ) + assert params["pgbouncer"] == "true" + + def test_build_db_connection_url_params_no_pgbouncer_by_default(self): + from litellm.proxy.proxy_cli import _build_db_connection_url_params + + params = _build_db_connection_url_params( + connection_limit=10, + pool_timeout=60, + ) + assert "pgbouncer" not in params + + def test_build_db_connection_url_params_extra_pgbouncer_overrides_flag(self): + from litellm.proxy.proxy_cli import _build_db_connection_url_params + + params = _build_db_connection_url_params( + connection_limit=10, + pool_timeout=60, + disable_prepared_statements=True, + extra_params={"pgbouncer": "false"}, + ) + assert params["pgbouncer"] == "false" + + @pytest.mark.parametrize( + "config_value, expect_pgbouncer", + [ + (True, True), + (False, False), + ("true", True), + ("false", False), + ("not-a-bool", False), + ], + ) + @patch("subprocess.run") + @patch("atexit.register") + @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") + @patch( + "litellm.proxy.db.prisma_client.should_update_prisma_schema", return_value=False + ) + def test_disable_prepared_statements_forwarded_to_url( + self, + mock_should_update, + mock_setup_db, + mock_atexit_register, + mock_subprocess_run, + config_value, + expect_pgbouncer, + ): + from click.testing import CliRunner + + from litellm.proxy.proxy_cli import run_server + + runner = CliRunner() + mock_subprocess_run.return_value = MagicMock(returncode=0) + + mock_proxy_module = MagicMock( + app=MagicMock(), + ProxyConfig=MagicMock(), + KeyManagementSettings=MagicMock(), + save_worker_config=MagicMock(), + ) + mock_proxy_module.ProxyConfig.return_value.get_config = AsyncMock( + return_value={ + "general_settings": { + "database_url": "postgresql://test:test@localhost:5432/test", + "database_disable_prepared_statements": config_value, + } + } + ) + + clean_env = { + k: v + for k, v in os.environ.items() + if k not in ("DATABASE_URL", "DIRECT_URL") + } + + with ( + patch.dict(os.environ, clean_env, clear=True), + patch.dict( + "sys.modules", + { + "proxy_server": mock_proxy_module, + "litellm.proxy.proxy_server": mock_proxy_module, + }, + ), + patch( + "litellm.proxy.proxy_cli.ProxyInitializationHelpers._get_default_unvicorn_init_args" + ) as mock_get_args, + patch( + "litellm.proxy.proxy_cli.append_query_params", + side_effect=lambda url, params: str(url), + ) as mock_append_query_params, + ): + mock_get_args.return_value = { + "app": "litellm.proxy.proxy_server:app", + "host": "localhost", + "port": 8000, + } + + result = runner.invoke( + run_server, + ["--local", "--config", "test-config.yaml", "--skip_server_startup"], + ) + + assert ( + result.exit_code == 0 + ), f"exit_code={result.exit_code}, output={result.output}" + mock_append_query_params.assert_called() + appended_params = mock_append_query_params.call_args.args[1] + if expect_pgbouncer: + assert appended_params["pgbouncer"] == "true" + else: + assert "pgbouncer" not in appended_params + @patch("uvicorn.run") @patch("atexit.register") @patch("litellm.proxy.db.prisma_client.PrismaManager.setup_database") diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index b912fec2479..19374205ba3 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3810,14 +3810,15 @@ async def test_model_info_v1_oci_secrets_not_leaked(): # Mock the llm_router to return our test data mock_router = MagicMock() + mock_router.model_list = [mock_model_data] mock_router.get_model_names.return_value = ["oci-grok-test"] mock_router.get_model_access_groups.return_value = {} - mock_router.get_model_list.return_value = [mock_model_data] # Mock global variables with ( patch("litellm.proxy.proxy_server.llm_router", mock_router), patch("litellm.proxy.proxy_server.llm_model_list", [mock_model_data]), + patch("litellm.proxy.proxy_server.prisma_client", None), patch( "litellm.proxy.proxy_server.general_settings", {"infer_model_from_keys": False}, diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/__init__.py b/tests/test_litellm/proxy/utils/prisma_and_spend/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py b/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py new file mode 100644 index 00000000000..2305a88b6dd --- /dev/null +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/conftest.py @@ -0,0 +1,387 @@ +"""Shared fixtures for tests/test_litellm/proxy/utils/prisma_and_spend/. + +All fixtures used by PR2 test files live here. Do NOT add fixtures inside +individual test files; if a fixture is missing, add it here and update the +Notion plan. + +The PrismaClient is exercised against a fully-mocked Prisma stack: the +``prisma.Prisma`` constructor and the writer/reader wrappers are patched +before PrismaClient.__init__ runs so the init code paths execute without +needing a generated Prisma client or a real database. +""" + +from __future__ import annotations + +import asyncio +import sys +from dataclasses import dataclass, field +from email.message import EmailMessage +from pathlib import Path +from typing import Any, Callable, Dict, Iterator, List, Optional +from unittest.mock import AsyncMock, MagicMock + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[5])) + + +VOLATILE_KEYS = frozenset( + { + "created_at", + "updated_at", + "checked_at", + "started_at", + "request_id", + "id", + "token", + "expires", + "expires_at", + "litellm_call_id", + "created", + "spend", + "last_refreshed_at", + "startTime", + "endTime", + "salt", + } +) + + +def normalize(data: Any, volatile: frozenset = VOLATILE_KEYS) -> Any: + """Recursively replace values for volatile keys with ''.""" + if isinstance(data, dict): + return { + k: ("" if k in volatile else normalize(v, volatile)) + for k, v in data.items() + } + if isinstance(data, list): + return [normalize(v, volatile) for v in data] + return data + + +_PRISMA_TABLES: List[str] = [ + "litellm_verificationtoken", + "litellm_teamtable", + "litellm_usertable", + "litellm_endusertable", + "litellm_organizationtable", + "litellm_proxymodeltable", + "litellm_modeltable", + "litellm_budgettable", + "litellm_spendlogs", + "litellm_config", + "litellm_usernotifications", + "litellm_healthchecktable", + "litellm_dailyuserspend", + "litellm_dailyteamspend", + "litellm_dailytagspend", + "litellm_managed_object_table", + "litellm_credentialstable", + "litellm_mcpservertable", + "litellm_audit_log", + "litellm_invitationlink", + "litellm_session_token_table", + "litellm_passthrough_endpoint_table", + "litellm_cron_job", + "litellm_passthrough_logs", + "litellm_promptstable", + "litellm_guardrailstable", + "litellm_managed_files", + "litellm_mcpusercredentials", + "litellm_objectpermissiontable", + "litellm_organizationmembership", +] + + +def _make_table_mock() -> MagicMock: + table = MagicMock() + table.find_unique = AsyncMock(return_value=None) + table.find_many = AsyncMock(return_value=[]) + table.find_first = AsyncMock(return_value=None) + table.create = AsyncMock() + table.create_many = AsyncMock() + table.update = AsyncMock() + table.update_many = AsyncMock() + table.upsert = AsyncMock() + table.delete = AsyncMock() + table.delete_many = AsyncMock() + table.count = AsyncMock(return_value=0) + table.group_by = AsyncMock(return_value=[]) + table.aggregate = AsyncMock(return_value={}) + return table + + +@pytest.fixture +def mock_prisma_client() -> MagicMock: + """Bare ``db`` mock with all common LiteLLM_* tables stubbed. + + Override individual return values in a test:: + + mock_prisma_client.db.litellm_usertable.find_unique.return_value = user + """ + client = MagicMock(name="MockPrismaClient") + client.db = MagicMock(name="MockPrismaDB") + client.connect = AsyncMock() + client.disconnect = AsyncMock() + client.health_check = AsyncMock(return_value=[{"?column?": 1}]) + client.proxy_logging_obj = MagicMock() + client.proxy_logging_obj.failure_handler = AsyncMock() + client.spend_log_transactions = [] + client._spend_log_transactions_lock = asyncio.Lock() + client.jsonify_object = lambda data: dict(data) + client.db.is_connected = MagicMock(return_value=False) + client.db.connect = AsyncMock() + client.db.disconnect = AsyncMock() + client.db.query_raw = AsyncMock(return_value=[{"?column?": 1}]) + client.db.execute_raw = AsyncMock() + client.db.tx = MagicMock() + client.db.batch_ = MagicMock() + for table_name in _PRISMA_TABLES: + setattr(client.db, table_name, _make_table_mock()) + return client + + +@pytest.fixture +def mock_dual_cache() -> MagicMock: + """In-memory DualCache stand-in. + + Sync and async get/set wired against a private dict. Override or read + ``cache._store`` directly in a test for assertion convenience. + """ + cache = MagicMock(name="MockDualCache") + cache._store: Dict[str, Any] = {} + + def _sync_get(key: str, **_: Any) -> Any: + return cache._store.get(key) + + def _sync_set(key: str, value: Any, **_: Any) -> None: + cache._store[key] = value + + async def _async_get(key: str, **_: Any) -> Any: + return cache._store.get(key) + + async def _async_set(key: str, value: Any, **_: Any) -> None: + cache._store[key] = value + + async def _async_delete(key: str, **_: Any) -> None: + cache._store.pop(key, None) + + cache.get_cache = MagicMock(side_effect=_sync_get) + cache.set_cache = MagicMock(side_effect=_sync_set) + cache.async_get_cache = AsyncMock(side_effect=_async_get) + cache.async_set_cache = AsyncMock(side_effect=_async_set) + cache.async_delete_cache = AsyncMock(side_effect=_async_delete) + return cache + + +@pytest.fixture +def patched_prisma_import(monkeypatch: pytest.MonkeyPatch) -> Iterator[MagicMock]: + """Replace ``prisma.Prisma`` and ``PrismaWrapper`` so PrismaClient.__init__ + runs without a generated client. Yields the fake Prisma instance. + + ``prisma`` raises RuntimeError (not AttributeError) for the missing + ``Prisma`` attribute, so ``monkeypatch.setattr`` can't probe it; assign + directly and restore in teardown. + """ + import prisma as _prisma_pkg + import litellm.proxy.utils as _utils_mod + + fake_prisma = MagicMock(name="FakePrisma") + fake_prisma.is_connected = MagicMock(return_value=False) + fake_prisma.connect = AsyncMock() + fake_prisma.disconnect = AsyncMock() + + fake_prisma_factory = MagicMock(name="FakePrismaFactory", return_value=fake_prisma) + had_prisma_attr = "Prisma" in _prisma_pkg.__dict__ + previous_prisma_attr = _prisma_pkg.__dict__.get("Prisma") + _prisma_pkg.Prisma = fake_prisma_factory # type: ignore[attr-defined] + + fake_wrapper = MagicMock(name="FakePrismaWrapper") + fake_wrapper.is_connected = MagicMock(return_value=False) + fake_wrapper.connect = AsyncMock() + fake_wrapper.disconnect = AsyncMock() + fake_wrapper.query_raw = AsyncMock(return_value=[{"?column?": 1}]) + + def _fake_wrapper_ctor(*args: Any, **kwargs: Any) -> MagicMock: + return fake_wrapper + + monkeypatch.setattr(_utils_mod, "PrismaWrapper", _fake_wrapper_ctor) + fake_prisma.__wrapper__ = fake_wrapper + try: + yield fake_prisma + finally: + if had_prisma_attr: + _prisma_pkg.Prisma = previous_prisma_attr # type: ignore[attr-defined] + else: + try: + del _prisma_pkg.Prisma # type: ignore[attr-defined] + except AttributeError: + pass + + +@pytest.fixture +def prisma_client( + patched_prisma_import: MagicMock, + mock_prisma_client: MagicMock, + monkeypatch: pytest.MonkeyPatch, +) -> Any: + """Wired ``PrismaClient`` whose ``db`` attribute is the table mock. + + The init runs through the real code path (testing the constructor's + config-attribute setup) and is then snapped to the easier-to-assert + table mock for downstream behavior pinning. + """ + monkeypatch.delenv("DATABASE_URL_READ_REPLICA", raising=False) + monkeypatch.delenv("IAM_TOKEN_DB_AUTH", raising=False) + from litellm.proxy.utils import PrismaClient + + proxy_logging_obj = MagicMock(name="MockProxyLogging") + proxy_logging_obj.failure_handler = AsyncMock() + pc = PrismaClient( + database_url="postgresql://test:test@localhost:5432/test", + proxy_logging_obj=proxy_logging_obj, + ) + pc.db = mock_prisma_client.db + return pc + + +@dataclass +class FakeClock: + """Monotonic-time controller for the spend monitor loop. + + Tests advance time via ``clock.advance(seconds)`` while asyncio.sleep + is replaced with a clock-driven no-op. + """ + + now: float = 0.0 + sleep_calls: List[float] = field(default_factory=list) + + def advance(self, seconds: float) -> None: + self.now += seconds + + def time(self) -> float: + return self.now + + async def sleep(self, seconds: float) -> None: + self.sleep_calls.append(seconds) + self.now += seconds + + +@pytest.fixture +def fake_clock(monkeypatch: pytest.MonkeyPatch) -> FakeClock: + """Install a controllable clock + asyncio.sleep replacement.""" + clock = FakeClock() + monkeypatch.setattr("time.time", clock.time) + monkeypatch.setattr("time.monotonic", clock.time) + + async def _fast_sleep(seconds: float, *_: Any, **__: Any) -> None: + clock.sleep_calls.append(seconds) + clock.now += seconds + + monkeypatch.setattr("asyncio.sleep", _fast_sleep) + return clock + + +@pytest.fixture +def make_spend_log_row() -> Callable[..., Dict[str, Any]]: + """Factory for fake LiteLLM_SpendLogs rows.""" + + def _make( + request_id: str = "req-1", + spend: float = 0.01, + model: str = "gpt-4o-mini", + **overrides: Any, + ) -> Dict[str, Any]: + row = { + "request_id": request_id, + "spend": spend, + "model": model, + "user": "user-1", + "team_id": "team-1", + "api_key": "hashed-key", + "startTime": "2026-06-02T00:00:00Z", + "endTime": "2026-06-02T00:00:01Z", + "metadata": {}, + } + row.update(overrides) + return row + + return _make + + +@dataclass +class _SentMessage: + from_addr: Optional[str] + to_addrs: Any + subject: Optional[str] + body: Optional[str] + starttls_called: bool + login_args: Optional[tuple] + + +@dataclass +class InMemorySMTP: + """Captures outbound SMTP traffic for ``send_email`` tests.""" + + sent: List[_SentMessage] = field(default_factory=list) + raise_on_send: Optional[Exception] = None + + def server_factory(self) -> Callable[..., Any]: + outer = self + + class _Conn: + def __init__(self) -> None: + self._starttls_called = False + self._login_args: Optional[tuple] = None + + def __enter__(self) -> "_Conn": + return self + + def __exit__(self, *exc: Any) -> None: + return None + + def starttls(self) -> None: + self._starttls_called = True + + def login(self, user: str, password: str) -> None: + self._login_args = (user, password) + + def send_message( + self, + msg: EmailMessage, + from_addr: Optional[str] = None, + to_addrs: Any = None, + ) -> None: + if outer.raise_on_send is not None: + raise outer.raise_on_send + body = "" + for part in msg.walk(): + if part.get_content_type() == "text/html": + body = part.get_payload(decode=False) or "" + break + outer.sent.append( + _SentMessage( + from_addr=from_addr, + to_addrs=to_addrs, + subject=msg["Subject"], + body=body, + starttls_called=self._starttls_called, + login_args=self._login_args, + ) + ) + + def _factory(*args: Any, **kwargs: Any) -> _Conn: + return _Conn() + + return _factory + + +@pytest.fixture +def in_memory_smtp(monkeypatch: pytest.MonkeyPatch) -> InMemorySMTP: + """Patch ``smtplib.SMTP`` to capture sends in memory. + + Override ``smtp.raise_on_send`` to test the SMTP error path. + """ + smtp = InMemorySMTP() + monkeypatch.setattr("smtplib.SMTP", smtp.server_factory()) + return smtp diff --git a/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py new file mode 100644 index 00000000000..08d1ef619a7 --- /dev/null +++ b/tests/test_litellm/proxy/utils/prisma_and_spend/test_prisma_client_get_data.py @@ -0,0 +1,516 @@ +"""Pin ``PrismaClient`` read-side data operations. + +Symbols pinned here: + - ``PrismaClient.hash_token`` + - ``PrismaClient.jsonify_object`` + - ``PrismaClient.jsonify_team_object`` + - ``PrismaClient.check_view_exists`` + - ``PrismaClient.get_request_status`` + - ``PrismaClient.get_generic_data`` + - ``PrismaClient._query_first_with_cached_plan_fallback`` + - ``PrismaClient.get_data`` +""" + +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import HTTPException + +from litellm.proxy._types import LiteLLM_VerificationTokenView +from litellm.proxy.utils import PrismaClient + + +def test_hash_token_method_returns_sha256(prisma_client: PrismaClient) -> None: + token = "sk-token-xyz" + actual = { + "result": prisma_client.hash_token(token), + "len": len(prisma_client.hash_token(token)), + "expected": hashlib.sha256(token.encode()).hexdigest(), + "deterministic": prisma_client.hash_token(token) + == prisma_client.hash_token(token), + } + assert actual == { + "result": hashlib.sha256(token.encode()).hexdigest(), + "len": 64, + "expected": hashlib.sha256(token.encode()).hexdigest(), + "deterministic": True, + } + + +def test_hash_token_method_error_on_non_string(prisma_client: PrismaClient) -> None: + with pytest.raises(AttributeError): + prisma_client.hash_token(None) # type: ignore[arg-type] + + +def test_jsonify_object_serializes_nested_dicts(prisma_client: PrismaClient) -> None: + data = { + "metadata": {"a": 1, "b": [2, 3]}, + "models": ["gpt-4o", "gpt-4o-mini"], + "token": "abc", + "spend": 1.23, + } + result = prisma_client.jsonify_object(data) + parsed_meta = json.loads(result["metadata"]) + assert result == { + "metadata": json.dumps(data["metadata"]), + "models": ["gpt-4o", "gpt-4o-mini"], + "token": "abc", + "spend": 1.23, + } + assert parsed_meta == {"a": 1, "b": [2, 3]} + + +def test_jsonify_object_fallback_for_unserializable_dict( + prisma_client: PrismaClient, +) -> None: + class _Bad: + pass + + data = {"metadata": {"x": _Bad()}, "label": "ok", "n": 1} + result = prisma_client.jsonify_object(data) + assert result == { + "metadata": "failed-to-serialize-json", + "label": "ok", + "n": 1, + } + + +def test_jsonify_object_error_on_non_dict(prisma_client: PrismaClient) -> None: + with pytest.raises(AttributeError): + prisma_client.jsonify_object(None) # type: ignore[arg-type] + + +def test_jsonify_team_object_converts_members_to_json_string( + prisma_client: PrismaClient, +) -> None: + data = { + "team_id": "t1", + "members_with_roles": [{"role": "admin", "user_id": "u1"}], + "metadata": {"foo": "bar"}, + "models": ["gpt-4"], + } + result = prisma_client.jsonify_team_object(data) + assert result == { + "team_id": "t1", + "members_with_roles": json.dumps(data["members_with_roles"]), + "metadata": json.dumps(data["metadata"]), + "models": ["gpt-4"], + } + + +def test_jsonify_team_object_error_on_non_dict(prisma_client: PrismaClient) -> None: + with pytest.raises(AttributeError): + prisma_client.jsonify_team_object(None) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "metadata,expected", + [ + ({"status": "failure"}, "failure"), + ({"status": "success"}, "success"), + ({}, "success"), + ("not-json", "success"), + (json.dumps({"status": "failure"}), "failure"), + ], +) +def test_get_request_status_pins_status_resolution( + prisma_client: PrismaClient, metadata: Any, expected: str +) -> None: + assert prisma_client.get_request_status({"metadata": metadata}) == expected + + +def test_get_request_status_error_returns_success_default( + prisma_client: PrismaClient, +) -> None: + """``get_request_status`` swallows AttributeError / JSONDecodeError and + defaults to ``success`` to avoid blocking the request pipeline. + """ + + class _Broken: + def get(self, *_: Any, **__: Any) -> Any: + raise AttributeError("broken metadata") + + actual = prisma_client.get_request_status({"metadata": _Broken()}) + assert actual == "success" + + +@pytest.mark.asyncio +async def test_get_generic_data_dispatches_by_table( + prisma_client: PrismaClient, +) -> None: + row = SimpleNamespace(user_id="u1", spend=0.5, name="Alice") + prisma_client.db.litellm_usertable.find_first = AsyncMock(return_value=row) + result = await prisma_client.get_generic_data( + key="user_id", value="u1", table_name="users" + ) + actual = { + "result_is_row": result is row, + "find_first_count": prisma_client.db.litellm_usertable.find_first.await_count, + "where_kwarg": prisma_client.db.litellm_usertable.find_first.await_args.kwargs[ + "where" + ], + "user_attr": result.user_id, + } + assert actual == { + "result_is_row": True, + "find_first_count": 1, + "where_kwarg": {"user_id": "u1"}, + "user_attr": "u1", + } + + +@pytest.mark.asyncio +async def test_get_generic_data_unknown_table_returns_none( + prisma_client: PrismaClient, +) -> None: + result = await prisma_client.get_generic_data( + key="x", value="y", table_name="bogus" # type: ignore[arg-type] + ) + assert result is None + + +@pytest.mark.asyncio +async def test_get_generic_data_logs_failure_handler_and_raises_on_error( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.litellm_usertable.find_first = AsyncMock( + side_effect=RuntimeError("db boom") + ) + with pytest.raises(RuntimeError, match="db boom"): + await prisma_client.get_generic_data( + key="user_id", value="x", table_name="users" + ) + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_happy_returns_row( + prisma_client: PrismaClient, +) -> None: + expected = {"token": "abc", "team_spend": 1.0, "team_max_budget": 5.0} + prisma_client.db.query_first = AsyncMock(return_value=expected) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) + result = await prisma_client._query_first_with_cached_plan_fallback( + "SELECT * FROM x WHERE token = $1", "abc" + ) + actual = { + "result": result, + "call_count": prisma_client.db.query_first.await_count, + "args": prisma_client.db.query_first.await_args.args, + "matches": result == expected, + } + assert actual == { + "result": expected, + "call_count": 1, + "args": ("SELECT * FROM x WHERE token = $1", "abc"), + "matches": True, + } + prisma_client.attempt_db_reconnect.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_reconnects_then_retries_identical_query( + prisma_client: PrismaClient, +) -> None: + original_query = 'SELECT * FROM "LiteLLM_VerificationToken" WHERE v.token = $1' + expected = {"token": "abc", "team_spend": 1.0, "team_max_budget": 5.0} + manager = MagicMock() + query_first = AsyncMock( + side_effect=[ + RuntimeError("cached plan must not change result type"), + expected, + ] + ) + reconnect = AsyncMock(return_value=True) + manager.attach_mock(query_first, "query_first") + manager.attach_mock(reconnect, "attempt_db_reconnect") + prisma_client.db.query_first = query_first + prisma_client.attempt_db_reconnect = reconnect + + result = await prisma_client._query_first_with_cached_plan_fallback( + original_query, "abc" + ) + + assert result == expected + assert query_first.await_count == 2 + first_call, retry_call = query_first.await_args_list + assert retry_call.args == first_call.args == (original_query, "abc") + reconnect.assert_awaited_once() + assert reconnect.await_args.kwargs.get("force", False) is False + assert [name for name, *_ in manager.mock_calls] == [ + "query_first", + "attempt_db_reconnect", + "query_first", + ] + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_never_deallocates( + prisma_client: PrismaClient, +) -> None: + expected = {"token": "abc"} + prisma_client.db.query_first = AsyncMock( + side_effect=[ + RuntimeError("cached plan must not change result type"), + expected, + ] + ) + prisma_client.db.execute_raw = AsyncMock(return_value=0) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) + + await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + + prisma_client.db.execute_raw.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_propagates_when_retry_also_fails( + prisma_client: PrismaClient, +) -> None: + plan_error = RuntimeError("cached plan must not change result type") + prisma_client.db.query_first = AsyncMock(side_effect=[plan_error, plan_error]) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) + + with pytest.raises(RuntimeError, match="cached plan must not change result type"): + await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + + assert prisma_client.db.query_first.await_count == 2 + prisma_client.attempt_db_reconnect.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_retries_when_reconnect_returns_false( + prisma_client: PrismaClient, +) -> None: + expected = {"token": "abc"} + prisma_client.db.query_first = AsyncMock( + side_effect=[ + RuntimeError("cached plan must not change result type"), + expected, + ] + ) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=False) + + result = await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + + assert result == expected + assert prisma_client.db.query_first.await_count == 2 + + +@pytest.mark.asyncio +async def test_query_first_with_cached_plan_fallback_reraises_non_plan_errors( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.query_first = AsyncMock( + side_effect=RuntimeError("totally unrelated") + ) + prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) + with pytest.raises(RuntimeError, match="totally unrelated"): + await prisma_client._query_first_with_cached_plan_fallback("SELECT 1") + assert prisma_client.db.query_first.await_count == 1 + prisma_client.attempt_db_reconnect.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_check_view_exists_noop_when_all_views_present( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.query_raw = AsyncMock( + return_value=[ + { + "view_count": 8, + "view_names": [ + "LiteLLM_VerificationTokenView", + "MonthlyGlobalSpend", + "Last30dKeysBySpend", + "Last30dModelsBySpend", + "MonthlyGlobalSpendPerKey", + "MonthlyGlobalSpendPerUserPerKey", + "Last30dTopEndUsersSpend", + "DailyTagSpend", + ], + } + ] + ) + prisma_client.db.execute_raw = AsyncMock() + result = await prisma_client.check_view_exists() + actual = { + "result": result, + "query_raw_calls": prisma_client.db.query_raw.await_count, + "execute_raw_calls": prisma_client.db.execute_raw.await_count, + "view_query_contains_token_view": "LiteLLM_VerificationTokenView" + in prisma_client.db.query_raw.await_args.args[0], + } + assert actual == { + "result": None, + "query_raw_calls": 1, + "execute_raw_calls": 0, + "view_query_contains_token_view": True, + } + + +@pytest.mark.asyncio +async def test_check_view_exists_creates_token_view_when_missing( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.query_raw = AsyncMock( + return_value=[ + { + "view_count": 1, + "view_names": ["DailyTagSpend"], + } + ] + ) + prisma_client.db.execute_raw = AsyncMock() + prisma_client.health_check = AsyncMock(return_value=[{"?column?": 1}]) + result = await prisma_client.check_view_exists() + actual = { + "result": result, + "create_called": prisma_client.db.execute_raw.await_count, + "create_sql_starts_with_create_view": prisma_client.db.execute_raw.await_args.args[ + 0 + ] + .strip() + .startswith('CREATE VIEW "LiteLLM_VerificationTokenView"'), + } + assert actual == { + "result": None, + "create_called": 1, + "create_sql_starts_with_create_view": True, + } + + +@pytest.mark.asyncio +async def test_check_view_exists_raises_when_query_raw_fails( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.query_raw = AsyncMock(side_effect=RuntimeError("db down")) + with pytest.raises(RuntimeError, match="db down"): + await prisma_client.check_view_exists() + + +@pytest.mark.asyncio +async def test_get_data_token_find_unique_returns_record( + prisma_client: PrismaClient, +) -> None: + token = "sk-key-1" + hashed = hashlib.sha256(token.encode()).hexdigest() + record = SimpleNamespace(token=hashed, user_id="u1", expires=None, spend=0.5) + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=record + ) + + result = await prisma_client.get_data(token=token, table_name="key") + actual = { + "result_is_record": result is record, + "where_arg": prisma_client.db.litellm_verificationtoken.find_unique.await_args.kwargs[ + "where" + ], + "include_arg": prisma_client.db.litellm_verificationtoken.find_unique.await_args.kwargs[ + "include" + ], + "token_field_matches": result.token == hashed, + } + assert actual == { + "result_is_record": True, + "where_arg": {"token": hashed}, + "include_arg": {"litellm_budget_table": True}, + "token_field_matches": True, + } + + +@pytest.mark.asyncio +async def test_get_data_token_find_unique_missing_token_raises_401( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + return_value=None + ) + with pytest.raises(HTTPException) as excinfo: + await prisma_client.get_data(token="sk-missing", table_name="key") + err = excinfo.value + assert "invalid user key" in err.detail + assert err.status_code == 401 + + +@pytest.mark.asyncio +async def test_get_data_user_find_unique_returns_user_row( + prisma_client: PrismaClient, +) -> None: + row = SimpleNamespace( + user_id="u-7", + spend=1.5, + max_budget=10.0, + organization_memberships=[], + ) + prisma_client.db.litellm_usertable.find_unique = AsyncMock(return_value=row) + result = await prisma_client.get_data(user_id="u-7", table_name="user") + actual = { + "result_is_row": result is row, + "where_arg": prisma_client.db.litellm_usertable.find_unique.await_args.kwargs[ + "where" + ], + "include_arg": prisma_client.db.litellm_usertable.find_unique.await_args.kwargs[ + "include" + ], + "spend": row.spend, + } + assert actual == { + "result_is_row": True, + "where_arg": {"user_id": "u-7"}, + "include_arg": {"organization_memberships": True}, + "spend": 1.5, + } + + +@pytest.mark.asyncio +async def test_get_data_logs_and_raises_on_db_error( + prisma_client: PrismaClient, +) -> None: + prisma_client.db.litellm_verificationtoken.find_unique = AsyncMock( + side_effect=RuntimeError("network split") + ) + with pytest.raises(RuntimeError, match="network split"): + await prisma_client.get_data(token="sk-broken", table_name="key") + + +@pytest.mark.asyncio +async def test_get_data_combined_view_returns_view_for_deprecated_key( + prisma_client: PrismaClient, +) -> None: + """Grace-period rotation, full get_data flow: the old hash misses the + combined view, the deprecated-key table resolves it to the active token, + and get_data must return the recursive lookup's finished view instead of + re-running dict normalization on it (which raised TypeError and turned + every grace-period request into a 401).""" + old_hash = "hashed-old-token-grace-e2e" + active_hash = "hashed-active-token-grace-e2e" + active_row = { + "token": active_hash, + "team_models": None, + "team_blocked": None, + "team_members_with_roles": None, + "user_id": None, + "expires": None, + } + prisma_client.db.query_first = AsyncMock(side_effect=[None, active_row]) + prisma_client.db.litellm_deprecatedverificationtoken = MagicMock() + prisma_client.db.litellm_deprecatedverificationtoken.find_first = AsyncMock( + return_value=SimpleNamespace( + active_token_id=active_hash, + revoke_at=datetime.now(timezone.utc) + timedelta(hours=1), + ) + ) + + response = await prisma_client.get_data( + token=old_hash, table_name="combined_view", query_type="find_unique" + ) + + assert isinstance(response, LiteLLM_VerificationTokenView) + assert response.token == active_hash diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index 18418f7bc78..726f1949d86 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -51,8 +51,8 @@ "@types/react-dom": "18.3.7", "@types/react-syntax-highlighter": "15.5.13", "@types/uuid": "10.0.0", - "@vitest/coverage-v8": "3.2.4", - "@vitest/ui": "3.2.4", + "@vitest/coverage-v8": "3.2.6", + "@vitest/ui": "3.2.6", "autoprefixer": "10.4.24", "dotenv": "17.2.3", "eslint": "9.39.2", @@ -66,7 +66,7 @@ "tailwindcss": "3.4.19", "typescript": "5.9.3", "vite": "7.3.2", - "vitest": "3.2.4" + "vitest": "3.2.6" }, "engines": { "node": ">=20.9.0", @@ -3998,9 +3998,9 @@ ] }, "node_modules/@vitest/coverage-v8": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-3.2.4.tgz", - "integrity": "sha512-EyF9SXU6kS5Ku/U82E259WSnvg6c8KTjppUncuNdm5QHpe17mwREHnjDzozC8x9MZ0xfBUFSaLkRv4TMA75ALQ==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/coverage-v8/-/coverage-v8-3.2.6.tgz", + "integrity": "sha512-LsAdmUapA0qSN306d8+zOyawM0hFm2m2Hg9IwVNIKBm+qJV8cijiq2c+gxKZcB1HCfIWAy+0qEZDCUQA58A1cw==", "dev": true, "license": "MIT", "dependencies": { @@ -4022,8 +4022,8 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "@vitest/browser": "3.2.4", - "vitest": "3.2.4" + "@vitest/browser": "3.2.6", + "vitest": "3.2.6" }, "peerDependenciesMeta": { "@vitest/browser": { @@ -4032,15 +4032,15 @@ } }, "node_modules/@vitest/expect": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.4.tgz", - "integrity": "sha512-Io0yyORnB6sikFlt8QW5K7slY4OjqNX9jmJQ02QDda8lyM6B5oNgVWoSoKPac8/kgnCUzuHQKrSLtu/uOqqrig==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.6.tgz", + "integrity": "sha512-1+7q9BtaKzEmO+fmNT3kYvoNn5Y71XWAx2Q5HRim4tTVRQVRv4uJFAQ5FbK0OPUeNP/WmVCpxYxoJdvuHVjzBQ==", "dev": true, "license": "MIT", "dependencies": { "@types/chai": "^5.2.2", - "@vitest/spy": "3.2.4", - "@vitest/utils": "3.2.4", + "@vitest/spy": "3.2.6", + "@vitest/utils": "3.2.6", "chai": "^5.2.0", "tinyrainbow": "^2.0.0" }, @@ -4049,13 +4049,13 @@ } }, "node_modules/@vitest/mocker": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.4.tgz", - "integrity": "sha512-46ryTE9RZO/rfDd7pEqFl7etuyzekzEhUbTW3BvmeO/BcCMEgq59BKhek3dXDWgAj4oMK6OZi+vRr1wPW6qjEQ==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-3.2.6.tgz", + "integrity": "sha512-EZOrpDbkKotFAP7wPAQV1UIyoGOk4oX7ynWhBhLB7v+meMHbQhU16oPpIYGTTe4oFlhpryGpgpcZP/sin3hYuw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/spy": "3.2.4", + "@vitest/spy": "3.2.6", "estree-walker": "^3.0.3", "magic-string": "^0.30.17" }, @@ -4076,9 +4076,9 @@ } }, "node_modules/@vitest/pretty-format": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.4.tgz", - "integrity": "sha512-IVNZik8IVRJRTr9fxlitMKeJeXFFFN0JaB9PHPGQ8NKQbGpfjlTx9zO4RefN8gp7eqjNy8nyK3NZmBzOPeIxtA==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/pretty-format/-/pretty-format-3.2.6.tgz", + "integrity": "sha512-lb7XXXzmm2h2ASzFnRvQpDo6onT1NmMJA3tkGTWiBFtRJ9lxGY3d3mm/Apt36gej2bkkOVLL/yTOtufDaFa/jA==", "dev": true, "license": "MIT", "dependencies": { @@ -4089,13 +4089,13 @@ } }, "node_modules/@vitest/runner": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.4.tgz", - "integrity": "sha512-oukfKT9Mk41LreEW09vt45f8wx7DordoWUZMYdY/cyAk7w5TWkTRCNZYF7sX7n2wB7jyGAl74OxgwhPgKaqDMQ==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/runner/-/runner-3.2.6.tgz", + "integrity": "sha512-HYcoSj1w5tcgUnzoF0HcyaAQjpA1gj9ftUJ7iSJSuipc02jW9gKkigwZbjFldAfYHA1fa8UZVRftdMY5msWM9Q==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "3.2.4", + "@vitest/utils": "3.2.6", "pathe": "^2.0.3", "strip-literal": "^3.0.0" }, @@ -4104,13 +4104,13 @@ } }, "node_modules/@vitest/snapshot": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.4.tgz", - "integrity": "sha512-dEYtS7qQP2CjU27QBC5oUOxLE/v5eLkGqPE0ZKEIDGMs4vKWe7IjgLOeauHsR0D5YuuycGRO5oSRXnwnmA78fQ==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/snapshot/-/snapshot-3.2.6.tgz", + "integrity": "sha512-H+ZjNTWGpObenh0YnlBctAPnJSI20P81PL8BPzWpx54YXLLTm8hEsWawtcYLMrwvpK48hGxLLbCS+1KRXhsKhw==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.4", + "@vitest/pretty-format": "3.2.6", "magic-string": "^0.30.17", "pathe": "^2.0.3" }, @@ -4119,9 +4119,9 @@ } }, "node_modules/@vitest/spy": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.4.tgz", - "integrity": "sha512-vAfasCOe6AIK70iP5UD11Ac4siNUNJ9i/9PZ3NKx07sG6sUxeag1LWdNrMWeKKYBLlzuK+Gn65Yd5nyL6ds+nw==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-3.2.6.tgz", + "integrity": "sha512-oq6BbH68WzcWmwtBrU9nqLeaXTR4XwJF7FSLkKEZo4i6eoXcrxjcwSuTvWBIRUTC6VC72nXYunzqgZA+IKdtxg==", "dev": true, "license": "MIT", "dependencies": { @@ -4132,13 +4132,13 @@ } }, "node_modules/@vitest/ui": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-3.2.4.tgz", - "integrity": "sha512-hGISOaP18plkzbWEcP/QvtRW1xDXF2+96HbEX6byqQhAUbiS5oH6/9JwW+QsQCIYON2bI6QZBF+2PvOmrRZ9wA==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/ui/-/ui-3.2.6.tgz", + "integrity": "sha512-mATfG3zVdhobE9U1rIpvtYD3DGuSSxqZ3Aj/8ityGqKXy8YDJ9BoAjZmAz6dZ1IZ1xI5V+MerkCczvVa+3QK9Q==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/utils": "3.2.4", + "@vitest/utils": "3.2.6", "fflate": "^0.8.2", "flatted": "^3.3.3", "pathe": "^2.0.3", @@ -4150,17 +4150,17 @@ "url": "https://opencollective.com/vitest" }, "peerDependencies": { - "vitest": "3.2.4" + "vitest": "3.2.6" } }, "node_modules/@vitest/utils": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.4.tgz", - "integrity": "sha512-fB2V0JFrQSMsCo9HiSq3Ezpdv4iYaXRG1Sx8edX3MwxfyNn83mKiGzOcH+Fkxt4MHxr3y42fQi1oeAInqgX2QA==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/@vitest/utils/-/utils-3.2.6.tgz", + "integrity": "sha512-lI23nIs4bnT3T8NIoh+vFaz5s2/DdP0Jgt2jxwgWljvwn82cLJtyi/If+fjFyoLMGIOz0U/fKvWE0d4jsNQEfg==", "dev": true, "license": "MIT", "dependencies": { - "@vitest/pretty-format": "3.2.4", + "@vitest/pretty-format": "3.2.6", "loupe": "^3.1.4", "tinyrainbow": "^2.0.0" }, @@ -4739,9 +4739,9 @@ } }, "node_modules/brace-expansion": { - "version": "5.0.5", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", - "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", + "version": "5.0.6", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.6.tgz", + "integrity": "sha512-kLpxurY4Z4r9sgMsyG0Z9uzsBlgiU/EFKhj/h91/8yHu0edo7XuixOIH3VcJ8kkxs6/jPzoI6U9Vj3WqbMQ94g==", "dev": true, "license": "MIT", "dependencies": { @@ -6512,9 +6512,9 @@ } }, "node_modules/fflate": { - "version": "0.8.2", - "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.2.tgz", - "integrity": "sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==", + "version": "0.8.3", + "resolved": "https://registry.npmjs.org/fflate/-/fflate-0.8.3.tgz", + "integrity": "sha512-tbZNuJrLwGUp3zshBtdy4W+ORxZuIh8a5ilyIEQDC5rY1f3U20JMry0Ll3WBzU58EZKsEuJFXhb5gwv8CsPvgA==", "dev": true, "license": "MIT" }, @@ -13002,20 +13002,20 @@ } }, "node_modules/vitest": { - "version": "3.2.4", - "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.4.tgz", - "integrity": "sha512-LUCP5ev3GURDysTWiP47wRRUpLKMOfPh+yKTx3kVIEiu5KOMeqzpnYNsKyOoVrULivR8tLcks4+lga33Whn90A==", + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/vitest/-/vitest-3.2.6.tgz", + "integrity": "sha512-xejya+bT/j/+R/AGa1XOfRxLmNUlLtlwjRsFUILF+xHfzElmGcmFydy2gqqIrd62ptIEfwVMofd19uNWD9L7Nw==", "dev": true, "license": "MIT", "dependencies": { "@types/chai": "^5.2.2", - "@vitest/expect": "3.2.4", - "@vitest/mocker": "3.2.4", - "@vitest/pretty-format": "^3.2.4", - "@vitest/runner": "3.2.4", - "@vitest/snapshot": "3.2.4", - "@vitest/spy": "3.2.4", - "@vitest/utils": "3.2.4", + "@vitest/expect": "3.2.6", + "@vitest/mocker": "3.2.6", + "@vitest/pretty-format": "^3.2.6", + "@vitest/runner": "3.2.6", + "@vitest/snapshot": "3.2.6", + "@vitest/spy": "3.2.6", + "@vitest/utils": "3.2.6", "chai": "^5.2.0", "debug": "^4.4.1", "expect-type": "^1.2.1", @@ -13045,8 +13045,8 @@ "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", - "@vitest/browser": "3.2.4", - "@vitest/ui": "3.2.4", + "@vitest/browser": "3.2.6", + "@vitest/ui": "3.2.6", "happy-dom": "*", "jsdom": "*" }, diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 6c37996ac09..ce793f97715 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -63,8 +63,8 @@ "@types/react-dom": "18.3.7", "@types/react-syntax-highlighter": "15.5.13", "@types/uuid": "10.0.0", - "@vitest/coverage-v8": "3.2.4", - "@vitest/ui": "3.2.4", + "@vitest/coverage-v8": "3.2.6", + "@vitest/ui": "3.2.6", "autoprefixer": "10.4.24", "dotenv": "17.2.3", "eslint": "9.39.2", @@ -78,7 +78,7 @@ "tailwindcss": "3.4.19", "typescript": "5.9.3", "vite": "7.3.2", - "vitest": "3.2.4" + "vitest": "3.2.6" }, "overrides": { "prismjs": "1.30.0", @@ -88,6 +88,7 @@ "lodash": "4.18.1", "ws": "8.20.1", "braces": "3.0.3", + "brace-expansion": "5.0.6", "axios": "1.13.6", "postcss": "8.5.13" }, diff --git a/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx b/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx index c47c201d074..8b0de5dda7c 100644 --- a/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx +++ b/ui/litellm-dashboard/src/components/playground/chat_ui/AgentBuilderView.tsx @@ -1,6 +1,14 @@ "use client"; -import { CommentOutlined, DeleteOutlined, ExperimentOutlined, LinkOutlined, PlusOutlined, RobotOutlined, SaveOutlined } from "@ant-design/icons"; +import { + CommentOutlined, + DeleteOutlined, + ExperimentOutlined, + LinkOutlined, + PlusOutlined, + RobotOutlined, + SaveOutlined, +} from "@ant-design/icons"; import { Button, Input, Modal, Select, Spin, Tabs } from "antd"; import React, { useCallback, useEffect, useState } from "react"; import CodeBlock from "@/app/(dashboard)/api-reference/components/CodeBlock"; @@ -64,9 +72,10 @@ function ConnectTabContent({ onCreateKey, }: ConnectTabContentProps) { const baseUrl = proxyBaseUrl ?? getConnectTabBaseUrl(proxySettings, customProxyBaseUrl); - const apiKeyForCurl = - createdKeyValue ? - createdKeyValue.startsWith("Bearer ") ? createdKeyValue : `Bearer ${createdKeyValue}` + const apiKeyForCurl = createdKeyValue + ? createdKeyValue.startsWith("Bearer ") + ? createdKeyValue + : `Bearer ${createdKeyValue}` : "Bearer sk-1234"; const curlExample = `curl -L -X POST '${baseUrl}/v1/chat/completions' \\ -H 'x-litellm-api-key: ${apiKeyForCurl}' \\ @@ -101,12 +110,7 @@ function ConnectTabContent({ Create a virtual key that can only call this agent. The key will be scoped to you (user_id) and restricted to the model {agentName}.

- {disabledPersonalKeyCreation && ( @@ -127,6 +131,14 @@ function getAgentModelId(agent: AgentModel): string | null { return info?.id ?? null; } +// Selection key that always resolves to a non-null string. Prefers the DB +// id (stable across renames and unique across teams) but falls back to +// `model_name` so config-file-defined agents — which have no `model_info.id` +// — remain selectable. +function getAgentSelectionKey(agent: AgentModel): string { + return getAgentModelId(agent) ?? agent.model_name; +} + function parseUnderlyingModel(litellmModel: string | undefined): string | undefined { if (!litellmModel || !litellmModel.startsWith("litellm_agent/")) return undefined; return litellmModel.slice("litellm_agent/".length) || undefined; @@ -191,22 +203,25 @@ export default function AgentBuilderView({ const [deleting, setDeleting] = useState(false); const effectiveApiKey = apiKey || accessToken || ""; - const selectedAgent = selectedId === NEW_AGENT_ID ? null : agentModels.find((a) => a.model_name === selectedId) ?? null; + const selectedAgent = + selectedId === NEW_AGENT_ID ? null : agentModels.find((a) => getAgentSelectionKey(a) === selectedId) ?? null; const isNewAgent = selectedId === NEW_AGENT_ID; const selectedAgentModelId = selectedAgent ? getAgentModelId(selectedAgent) : null; - const loadAgents = useCallback(async () => { - if (!accessToken || !userID || !userRole) return; + const loadAgents = useCallback(async (): Promise => { + if (!accessToken || !userID || !userRole) return []; setLoadingAgents(true); try { const list = await fetchAvailableAgentModels(accessToken, userID, userRole); setAgentModels(list); - if (!selectedId || (selectedId !== NEW_AGENT_ID && !list.some((a) => a.model_name === selectedId))) { - setSelectedId(list.length > 0 ? list[0].model_name : null); + if (!selectedId || (selectedId !== NEW_AGENT_ID && !list.some((a) => getAgentSelectionKey(a) === selectedId))) { + setSelectedId(list.length > 0 ? getAgentSelectionKey(list[0]) : null); } + return list; } catch (e) { console.error(e); NotificationsManager.fromBackend("Failed to load agents"); + return []; } finally { setLoadingAgents(false); } @@ -267,7 +282,13 @@ export default function AgentBuilderView({ setDraftMaxTokens(typeof p?.max_tokens === "number" ? p.max_tokens : 4096); const rawTools = selectedAgent.litellm_params?.tools; const tools: MCPToolEntry[] = Array.isArray(rawTools) - ? rawTools.filter((t): t is MCPToolEntry => t && typeof t === "object" && (t as MCPToolEntry).type === "mcp" && typeof (t as MCPToolEntry).server_url === "string") + ? rawTools.filter( + (t): t is MCPToolEntry => + t && + typeof t === "object" && + (t as MCPToolEntry).type === "mcp" && + typeof (t as MCPToolEntry).server_url === "string", + ) : []; setDraftTools(tools); } @@ -297,7 +318,7 @@ export default function AgentBuilderView({ } setSaving(true); try { - await modelCreateCall(accessToken, { + const response = await modelCreateCall(accessToken, { model_name: draftName.trim(), litellm_params: { model: `litellm_agent/${draftUnderlyingModel}`, @@ -308,9 +329,15 @@ export default function AgentBuilderView({ }, model_info: {}, }); - const newName = draftName.trim(); - await loadAgents(); - setSelectedId(newName); + // /model/new returns the row with `model_id` at the top level. + // Prefer that id over name-matching so we land on the just-created + // agent even when its public name collides with another team's. + const createdId: string | null = response?.model_id ?? response?.model_info?.id ?? null; + const list = await loadAgents(); + const created = createdId + ? list.find((a) => getAgentModelId(a) === createdId) ?? list.find((a) => a.model_name === draftName.trim()) + : list.find((a) => a.model_name === draftName.trim()); + setSelectedId(created ? getAgentSelectionKey(created) : list[0] ? getAgentSelectionKey(list[0]) : null); setActiveTab("chat"); } catch (e) { NotificationsManager.fromBackend("Failed to save agent"); @@ -342,8 +369,10 @@ export default function AgentBuilderView({ selectedAgentModelId, ); NotificationsManager.success("Agent updated successfully"); - await loadAgents(); - setSelectedId(draftName.trim()); + const list = await loadAgents(); + const stillSelected = list.find((a) => getAgentModelId(a) === selectedAgentModelId); + const target = stillSelected ?? list[0]; + setSelectedId(target ? getAgentSelectionKey(target) : null); } catch (e) { NotificationsManager.fromBackend("Failed to update agent"); } finally { @@ -387,9 +416,9 @@ export default function AgentBuilderView({ try { await modelDeleteCall(accessToken, selectedAgentModelId); NotificationsManager.success("Agent deleted"); - await loadAgents(); - const remaining = agentModels.filter((a) => a.model_name !== selectedAgent.model_name); - setSelectedId(remaining.length > 0 ? remaining[0].model_name : null); + const list = await loadAgents(); + const remaining = list.filter((a) => getAgentModelId(a) !== selectedAgentModelId); + setSelectedId(remaining.length > 0 ? getAgentSelectionKey(remaining[0]) : null); } catch (e) { NotificationsManager.fromBackend("Failed to delete agent"); } finally { @@ -401,9 +430,7 @@ export default function AgentBuilderView({ if (!accessToken || !userID || !userRole) { return ( -
- Sign in to use Agent Builder. -
+
Sign in to use Agent Builder.
); } @@ -412,24 +439,25 @@ export default function AgentBuilderView({
Agent Builder - {isNewAgent ? ( - - ) : ( - Build Agents that pass your compliance requirements. - )} + {isNewAgent ? ( + + ) : ( + Build Agents that pass your compliance requirements. + )}
- Agent Builder is experimental and may change or be removed without notice. We’d love your feedback—email us at{" "} + Agent Builder is experimental and may change or be removed without notice. We’d love your feedback—email us + at{" "} product@berri.ai @@ -452,21 +480,24 @@ export default function AgentBuilderView({
) : ( <> - {agentModels.map((agent) => ( - - ))} + {agentModels.map((agent) => { + const key = getAgentSelectionKey(agent); + return ( + + ); + })}