mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge b2a9d56bdd into 7a23e497bd
This commit is contained in:
commit
f53b38d8ea
6 changed files with 1381 additions and 140 deletions
|
|
@ -45,6 +45,7 @@ from litellm.constants import (
|
|||
MAX_LITELLM_CALL_ID_LENGTH,
|
||||
MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG,
|
||||
NON_INFERENCE_CALL_TYPES,
|
||||
REDACTED_BY_LITELLM,
|
||||
RETURN_RAW_MODEL_NAME_METADATA_KEY,
|
||||
STREAM_SSE_DATA_PREFIX,
|
||||
STREAM_SSE_KEEPALIVE_PING_BYTES,
|
||||
|
|
@ -128,6 +129,8 @@ from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _ar
|
|||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.proxy.route_llm_request import (
|
||||
can_route_with_loaded_models,
|
||||
raise_if_model_not_routable,
|
||||
route_request,
|
||||
)
|
||||
from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
|
||||
|
|
@ -275,6 +278,7 @@ from litellm.proxy.litellm_pre_call_utils import (
|
|||
from litellm.proxy.policy_engine.response_retrieval import attach_post_call_pipelines_to_retrieval
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.utils import (
|
||||
CallTypesLiteral,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StandardLoggingPayloadErrorInformation,
|
||||
|
|
@ -2008,9 +2012,10 @@ def _content_text(fields: Mapping[str, object]) -> str:
|
|||
|
||||
class ProxyBaseLLMRequestProcessing:
|
||||
def __init__(self, data: dict, served_model: str | None = None):
|
||||
self.data = data
|
||||
self.data: dict[str, Any] = data # mutable-ok: the request body every later step mutates in place
|
||||
self.served_model = served_model
|
||||
self._tags_before_guardrails: frozenset[str] | None = None
|
||||
self._rejected_before_guardrails = False
|
||||
self.route_type: str | None = None
|
||||
|
||||
@property
|
||||
|
|
@ -2501,14 +2506,27 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
if self._tags_before_guardrails is None:
|
||||
self._tags_before_guardrails = frozenset(get_tags_from_request_body(request_body=self.data))
|
||||
model_routable: Final = (
|
||||
skip_guardrails
|
||||
or llm_router is None
|
||||
or can_route_with_loaded_models(
|
||||
data=self.data,
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
route_type=route_type,
|
||||
)
|
||||
)
|
||||
prefetch_model = self.data.get("model")
|
||||
if llm_router is not None and isinstance(prefetch_model, str):
|
||||
llm_router.arm_routing_read_prefetch(prefetch_model, self.data)
|
||||
self.data = await proxy_logging_obj.pre_call_hook(
|
||||
await self._run_pre_call_hooks(
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
data=self.data,
|
||||
call_type=route_type,
|
||||
skip_guardrails=skip_guardrails,
|
||||
model_routable=model_routable,
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
)
|
||||
await _enforce_guardrail_added_tag_budgets(
|
||||
data=self.data,
|
||||
|
|
@ -2544,6 +2562,103 @@ class ProxyBaseLLMRequestProcessing:
|
|||
|
||||
return self.data, logging_obj
|
||||
|
||||
async def _run_pre_call_hooks(
|
||||
self,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
call_type: CallTypesLiteral,
|
||||
skip_guardrails: bool,
|
||||
model_routable: bool,
|
||||
llm_router: Router | None,
|
||||
user_model: str | None,
|
||||
) -> None:
|
||||
"""Run the pre-call hooks on self.data, checking a model that cannot route yet before the first guardrail.
|
||||
|
||||
For such a model, the hooks that come before the first guardrail or content enforcer run
|
||||
first and may rewrite the model. A model that still cannot route raises the routing error
|
||||
there, before any guardrail runs, otherwise the remaining hooks and guardrails run in their
|
||||
usual order. A request rejected before the check passed is redacted.
|
||||
"""
|
||||
if model_routable:
|
||||
self.data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
data=self.data,
|
||||
call_type=call_type,
|
||||
skip_guardrails=skip_guardrails,
|
||||
)
|
||||
return
|
||||
passed_routing_check: Final = list[bool]()
|
||||
|
||||
async def raise_unless_routable(request: dict[str, object]) -> None: # mutable-ok: request payload
|
||||
self.data = request
|
||||
try:
|
||||
await raise_if_model_not_routable(
|
||||
data=request,
|
||||
llm_router=llm_router,
|
||||
user_model=user_model,
|
||||
route_type=call_type,
|
||||
)
|
||||
except Exception:
|
||||
refresh_proxy_server_request_body_snapshot(request)
|
||||
raise
|
||||
passed_routing_check.append(True)
|
||||
|
||||
try:
|
||||
self.data = await proxy_logging_obj.pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
data=self.data,
|
||||
call_type=call_type,
|
||||
before_guardrails=raise_unless_routable,
|
||||
)
|
||||
except Exception:
|
||||
if not passed_routing_check:
|
||||
self._redact_request_rejected_before_guardrails(proxy_logging_obj=proxy_logging_obj)
|
||||
raise
|
||||
|
||||
def _redact_request_rejected_before_guardrails(self, proxy_logging_obj: ProxyLogging) -> None:
|
||||
"""Redact the content of a request rejected before the pre-call guardrails that apply to it ran.
|
||||
|
||||
Covers every request field a pre-call guardrail can rewrite on the routes that reach this
|
||||
check: chat and Anthropic messages and system, completion prompt, Responses input and
|
||||
instructions, embedding and speech input, tool definitions, and Gemini contents and system
|
||||
instruction. The request body, its proxy_server_request snapshot and message logging all
|
||||
carry the redaction marker, so failure callbacks, post-call failure hooks and spend rows
|
||||
never see content a masking guardrail would have rewritten.
|
||||
"""
|
||||
request_metadata: Final = dict[str, object]()
|
||||
for bucket in _request_metadata_buckets(self.data):
|
||||
request_metadata.update(bucket)
|
||||
if not proxy_logging_obj.has_pre_call_guardrails(request_metadata, request_body=self.data):
|
||||
return
|
||||
self._rejected_before_guardrails = True
|
||||
if "messages" in self.data:
|
||||
self.data["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
|
||||
if "contents" in self.data:
|
||||
self.data["contents"] = [{"role": "user", "parts": [{"text": REDACTED_BY_LITELLM}]}]
|
||||
for content_key in ("prompt", "input"):
|
||||
if content_key in self.data:
|
||||
self.data[content_key] = ""
|
||||
if "tools" in self.data:
|
||||
self.data["tools"] = []
|
||||
for content_key in ("system", "instructions"):
|
||||
if content_key in self.data:
|
||||
self.data[content_key] = REDACTED_BY_LITELLM
|
||||
redacted_system_instruction: Final = {"parts": [{"text": REDACTED_BY_LITELLM}]}
|
||||
system_instruction_keys: Final = ("systemInstruction", "system_instruction")
|
||||
for content_key in system_instruction_keys:
|
||||
if content_key in self.data:
|
||||
self.data[content_key] = redacted_system_instruction
|
||||
config: Final = self.data.get("config")
|
||||
if isinstance(config, Mapping):
|
||||
self.data["config"] = {
|
||||
**config,
|
||||
**{key: redacted_system_instruction for key in system_instruction_keys if key in config},
|
||||
}
|
||||
refresh_proxy_server_request_body_snapshot(self.data)
|
||||
logging_obj: Final = self.data.get("litellm_logging_obj")
|
||||
if isinstance(logging_obj, LiteLLMLoggingObj):
|
||||
logging_obj.standard_callback_dynamic_params["turn_off_message_logging"] = True
|
||||
|
||||
async def _pre_call_with_fallbacks(
|
||||
self,
|
||||
request: Request,
|
||||
|
|
@ -2638,6 +2753,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
):
|
||||
continue
|
||||
self.data = independent_snapshot(pristine)
|
||||
self._rejected_before_guardrails = False
|
||||
self.data["model"] = fallback_model
|
||||
try:
|
||||
return await self.common_processing_pre_call_logic(
|
||||
|
|
@ -2662,7 +2778,8 @@ class ProxyBaseLLMRequestProcessing:
|
|||
raise
|
||||
continue
|
||||
except BaseException:
|
||||
self.data = rate_limited_data
|
||||
if not self._rejected_before_guardrails:
|
||||
self.data = rate_limited_data
|
||||
raise
|
||||
|
||||
self.data = rate_limited_data
|
||||
|
|
@ -2783,7 +2900,7 @@ class ProxyBaseLLMRequestProcessing:
|
|||
"Request received by LiteLLM: payload too large to log (%d bytes, limit %d). Keys: %s",
|
||||
len(_payload_str),
|
||||
MAX_PAYLOAD_SIZE_FOR_DEBUG_LOG,
|
||||
(list(self.data.keys()) if isinstance(self.data, dict) else type(self.data).__name__),
|
||||
sanitize_for_log(list(self.data.keys())),
|
||||
)
|
||||
else:
|
||||
verbose_proxy_logger.debug(
|
||||
|
|
|
|||
|
|
@ -2,16 +2,18 @@ import asyncio
|
|||
import hashlib
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable, Mapping
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from enum import Enum, auto
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal
|
||||
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, TypeVar
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from fastapi import HTTPException, status
|
||||
from pydantic import BeforeValidator, TypeAdapter, ValidationError
|
||||
from typing_extensions import Never
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
|
|
@ -44,6 +46,8 @@ if TYPE_CHECKING:
|
|||
else:
|
||||
LitellmRouter = Any
|
||||
|
||||
_T: Final = TypeVar("_T")
|
||||
|
||||
|
||||
_PER_REQUEST_ROUTER_SETTINGS: Final = (
|
||||
"fallbacks",
|
||||
|
|
@ -721,15 +725,48 @@ async def route_request(
|
|||
Common helper to route the request
|
||||
"""
|
||||
router_only_settings_from_override: Final = _merge_router_settings_override(data) # pyright: ignore[reportUnknownArgumentType] # route_request takes an untyped dict body
|
||||
try:
|
||||
return await _route_request_single_attempt(
|
||||
return await _retry_once_after_model_read_through(
|
||||
data,
|
||||
llm_router,
|
||||
lambda router: _route_request_single_attempt( # pyright: ignore[reportUnknownLambdaType] # the router methods it returns are untyped
|
||||
data=data,
|
||||
llm_router=llm_router,
|
||||
llm_router=router,
|
||||
user_model=user_model,
|
||||
route_type=route_type,
|
||||
router_only_settings_from_override=router_only_settings_from_override,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
async def raise_if_model_not_routable(
|
||||
data: Mapping[str, object],
|
||||
llm_router: LitellmRouter | None,
|
||||
user_model: str | None,
|
||||
route_type: str,
|
||||
) -> None:
|
||||
"""Raise the error route_request would raise before sending this request, without sending it."""
|
||||
_merge_router_settings_override(dict(data))
|
||||
|
||||
async def check(router: LitellmRouter | None) -> None:
|
||||
raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=router)
|
||||
raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed())
|
||||
if (
|
||||
_decide_dispatch(data=data, llm_router=router, user_model=user_model, route_type=route_type)
|
||||
is _Dispatch.MODEL_NOT_FOUND
|
||||
):
|
||||
_raise_model_not_found(data=data, route_type=route_type)
|
||||
|
||||
await _retry_once_after_model_read_through(data, llm_router, check)
|
||||
|
||||
|
||||
async def _retry_once_after_model_read_through(
|
||||
data: Mapping[str, object],
|
||||
llm_router: LitellmRouter | None,
|
||||
attempt: Callable[[LitellmRouter | None], Awaitable[_T]],
|
||||
) -> _T:
|
||||
try:
|
||||
return await attempt(llm_router)
|
||||
except (ProxyModelNotFoundError, ProxyMissingParamWithoutLoadedModelError) as e:
|
||||
requested_model: Final = data.get("model", "")
|
||||
if (
|
||||
|
|
@ -745,73 +782,67 @@ async def route_request(
|
|||
|
||||
if not await model_registry_read_through.attempt(requested_model):
|
||||
raise
|
||||
return await _route_request_single_attempt(
|
||||
data=data,
|
||||
llm_router=proxy_server.llm_router,
|
||||
user_model=user_model,
|
||||
route_type=route_type,
|
||||
router_only_settings_from_override=router_only_settings_from_override,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
)
|
||||
return await attempt(proxy_server.llm_router)
|
||||
|
||||
|
||||
async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed
|
||||
data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request
|
||||
def _raise_model_not_found(data: Mapping[str, object], route_type: str) -> Never:
|
||||
# if no route found then it's a bad request
|
||||
route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
|
||||
raise ProxyModelNotFoundError(
|
||||
route=route_name,
|
||||
model_name=str(data.get("model", "")),
|
||||
)
|
||||
|
||||
|
||||
class _Dispatch(Enum):
|
||||
ROUTER = auto()
|
||||
ROUTER_SPECIFIC_DEPLOYMENT = auto()
|
||||
ROUTER_THEN_LITELLM = auto()
|
||||
LITELLM = auto()
|
||||
BATCH_COMPLETION = auto()
|
||||
USER_CONFIG = auto()
|
||||
EVAL = auto()
|
||||
A2A_AGENT = auto()
|
||||
MISSING_MODEL = auto()
|
||||
MODEL_NOT_FOUND = auto()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _TeamModel:
|
||||
name: str
|
||||
|
||||
|
||||
def _decide_dispatch( # noqa: C901 # one branch per dispatch target, moved out of _route_request_single_attempt
|
||||
data: Mapping[str, object],
|
||||
llm_router: LitellmRouter | None,
|
||||
user_model: str | None,
|
||||
route_type: RouteType,
|
||||
router_only_settings_from_override: frozenset[str],
|
||||
user_api_key_dict: UserAPIKeyAuth | None = None,
|
||||
):
|
||||
raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=llm_router)
|
||||
route_type: str,
|
||||
) -> _Dispatch | _TeamModel:
|
||||
"""Where route_request sends this request, decided without changing it.
|
||||
|
||||
await add_shared_session_to_data(data)
|
||||
|
||||
raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed())
|
||||
|
||||
def direct_litellm_kwargs() -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in JSON_OBJECT_ADAPTER.validate_python(data).items()
|
||||
if key not in router_only_settings_from_override
|
||||
}
|
||||
)
|
||||
|
||||
team_id: Final = get_team_id_from_data(data)
|
||||
Raises PermissionDeniedError (403) when every deployment of the requested model is blocked.
|
||||
"""
|
||||
model: Final = data.get("model")
|
||||
team_id: Final = get_team_id_from_data(dict(data))
|
||||
router_model_names: Final = llm_router.model_names if llm_router is not None else []
|
||||
is_proxy_admin_without_team: Final = team_id is None and is_proxy_admin_request(data)
|
||||
|
||||
# Preprocess Google GenAI generate content requests
|
||||
if route_type in ["agenerate_content", "agenerate_content_stream"]:
|
||||
# Map generationConfig to config parameter for Google GenAI compatibility
|
||||
if "generationConfig" in data and "config" not in data:
|
||||
data["config"] = data.pop("generationConfig")
|
||||
if "api_key" in data or "api_base" in data:
|
||||
if llm_router is not None:
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
return _Dispatch.ROUTER
|
||||
else:
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
return _Dispatch.LITELLM
|
||||
|
||||
elif (
|
||||
route_type == "acompletion"
|
||||
and data.get("model", "") is not None
|
||||
and "," in data.get("model", "")
|
||||
and llm_router is not None
|
||||
):
|
||||
elif route_type == "acompletion" and isinstance(model, str) and "," in model and llm_router is not None:
|
||||
# Handle batch completions with comma-separated models BEFORE user_config check
|
||||
# This ensures batch completion logic is applied even when user_config is set
|
||||
if data.get("fastest_response", False):
|
||||
return llm_router.abatch_completion_fastest_response(**data)
|
||||
else:
|
||||
models: Final = [model.strip() for model in data.pop("model").split(",")]
|
||||
return llm_router.abatch_completion(models=models, **data)
|
||||
return _Dispatch.BATCH_COMPLETION
|
||||
|
||||
elif "user_config" in data:
|
||||
return _route_user_config_request(data, route_type)
|
||||
elif data.get("user_config") is not None:
|
||||
return _Dispatch.USER_CONFIG
|
||||
|
||||
elif llm_router is not None:
|
||||
_raise_if_model_fully_blocked(llm_router=llm_router, model_name=data.get("model"), team_id=team_id)
|
||||
_raise_if_model_fully_blocked(llm_router=llm_router, model_name=model, team_id=team_id)
|
||||
# Evals API: always route to litellm directly (not through router)
|
||||
# But extract model credentials if a model is provided
|
||||
if route_type in [
|
||||
|
|
@ -827,30 +858,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
"acancel_run",
|
||||
"adelete_run",
|
||||
]:
|
||||
# If a model is provided, get its credentials from the router
|
||||
model: Final = data.get("model")
|
||||
if model and llm_router:
|
||||
try:
|
||||
# Try to get deployment credentials for this model
|
||||
deployment_creds = llm_router.get_deployment_credentials(model_id=model)
|
||||
if not deployment_creds:
|
||||
# Try by model group name
|
||||
deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model)
|
||||
if (
|
||||
deployment
|
||||
and deployment.litellm_params
|
||||
and not llm_router.is_deployment_blocked(deployment)
|
||||
):
|
||||
deployment_creds = deployment.litellm_params.model_dump(exclude_none=True)
|
||||
|
||||
# If we found credentials, merge them into data (but don't override user-provided values)
|
||||
if deployment_creds:
|
||||
data.update(deployment_creds)
|
||||
except Exception:
|
||||
# If we can't get deployment creds, continue without them
|
||||
pass
|
||||
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
return _Dispatch.EVAL
|
||||
# Skip model-based routing for container operations
|
||||
if route_type in [
|
||||
"acreate_container",
|
||||
|
|
@ -863,7 +871,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
"adelete_container_file",
|
||||
"aretrieve_container_file_content",
|
||||
]:
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
return _Dispatch.ROUTER
|
||||
# Interactions API: create with agent, get/delete/cancel don't need model routing
|
||||
if route_type in [
|
||||
"acreate_interaction",
|
||||
|
|
@ -871,7 +879,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
"adelete_interaction",
|
||||
"acancel_interaction",
|
||||
]:
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
return _Dispatch.ROUTER
|
||||
# Managed Agents API: these don't need model routing
|
||||
if route_type in [
|
||||
"acreate_agent",
|
||||
|
|
@ -880,7 +888,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
"adelete_agent",
|
||||
"alist_agent_versions",
|
||||
]:
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
return _Dispatch.ROUTER
|
||||
if route_type in [
|
||||
"avideo_list",
|
||||
"avideo_status",
|
||||
|
|
@ -899,17 +907,14 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
"aget_skill",
|
||||
"adelete_skill",
|
||||
"aingest",
|
||||
] and (data.get("model") is None or data.get("model") == ""):
|
||||
] and (model is None or model == ""):
|
||||
# These endpoints don't need a model, use custom_llm_provider directly
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
return _Dispatch.LITELLM
|
||||
|
||||
if "model" not in data:
|
||||
raise ProxyMissingRequiredParamError(
|
||||
route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type),
|
||||
param="model",
|
||||
)
|
||||
custom_llm_provider: Final[object] = (
|
||||
JSON_OBJECT_ADAPTER.validate_python(data).get("custom_llm_provider")
|
||||
return _Dispatch.MISSING_MODEL
|
||||
custom_llm_provider: Final = (
|
||||
data.get("custom_llm_provider")
|
||||
if route_type
|
||||
in (
|
||||
"avector_store_search",
|
||||
|
|
@ -921,31 +926,33 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
)
|
||||
team_model_provider: Final[str | None] = custom_llm_provider if isinstance(custom_llm_provider, str) else None
|
||||
team_model_name: Final = (
|
||||
llm_router.map_team_model(data["model"], team_id, custom_llm_provider=team_model_provider)
|
||||
if team_id is not None
|
||||
llm_router.map_team_model(model, team_id, custom_llm_provider=team_model_provider)
|
||||
if team_id is not None and (model is None or isinstance(model, str))
|
||||
else None
|
||||
)
|
||||
if team_model_name is not None:
|
||||
data["model"] = team_model_name
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
return _TeamModel(name=team_model_name)
|
||||
|
||||
elif (
|
||||
is_proxy_admin_without_team
|
||||
and data["model"] not in router_model_names
|
||||
and data["model"] in llm_router.team_public_model_names
|
||||
) or llm_router.is_recognized_model(data["model"]):
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
elif isinstance(model, str) and (
|
||||
(
|
||||
is_proxy_admin_without_team
|
||||
and model not in router_model_names
|
||||
and model in llm_router.team_public_model_names
|
||||
)
|
||||
or llm_router.is_recognized_model(model)
|
||||
):
|
||||
return _Dispatch.ROUTER
|
||||
|
||||
elif data["model"] not in router_model_names:
|
||||
elif model not in router_model_names:
|
||||
# Check wildcards before checking deployment_names
|
||||
# Priority: 1. Exact model_name match, 2. Wildcard match, 3. deployment_names match
|
||||
if llm_router.router_general_settings.pass_through_all_models:
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
return _Dispatch.LITELLM
|
||||
elif llm_router.default_deployment is not None or len(llm_router.pattern_router.patterns) > 0:
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
elif data["model"] in llm_router.deployment_names:
|
||||
return _Dispatch.ROUTER
|
||||
elif model in llm_router.deployment_names:
|
||||
# Only match deployment_names if no wildcard matched
|
||||
return getattr(llm_router, f"{route_type}")(**data, specific_deployment=True)
|
||||
return _Dispatch.ROUTER_SPECIFIC_DEPLOYMENT
|
||||
elif route_type in [
|
||||
"amoderation",
|
||||
"aget_responses",
|
||||
|
|
@ -976,7 +983,7 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
"aretrieve_container_file_content",
|
||||
]:
|
||||
# These endpoints can work with or without model parameter
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
return _Dispatch.ROUTER
|
||||
elif route_type in [
|
||||
"avideo_status",
|
||||
"avideo_content",
|
||||
|
|
@ -988,27 +995,126 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr
|
|||
]:
|
||||
# Video endpoints: If model is provided (e.g., from decoded video_id or target_model_names),
|
||||
# try router first to allow for multi-deployment load balancing
|
||||
try:
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
except Exception:
|
||||
# If router fails (e.g., model not found in router), fall back to direct call
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
elif _is_a2a_agent_model(data.get("model", "")):
|
||||
from litellm.proxy.agent_endpoints.a2a_routing import (
|
||||
route_a2a_agent_request,
|
||||
)
|
||||
|
||||
result: Final = await route_a2a_agent_request(data, route_type, user_api_key_dict=user_api_key_dict)
|
||||
if result is not None:
|
||||
return result
|
||||
# Fall through to raise exception below if result is None
|
||||
return _Dispatch.ROUTER_THEN_LITELLM
|
||||
elif _is_a2a_agent_model(model):
|
||||
return _Dispatch.A2A_AGENT
|
||||
|
||||
elif user_model is not None or route_type == "allm_passthrough_route":
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
return _Dispatch.LITELLM
|
||||
|
||||
# if no route found then it's a bad request
|
||||
route_name: Final = ROUTE_ENDPOINT_MAPPING.get(route_type, route_type)
|
||||
raise ProxyModelNotFoundError(
|
||||
route=route_name,
|
||||
model_name=data.get("model", ""),
|
||||
)
|
||||
return _Dispatch.MODEL_NOT_FOUND
|
||||
|
||||
|
||||
def can_route_with_loaded_models(
|
||||
data: Mapping[str, object],
|
||||
llm_router: LitellmRouter | None,
|
||||
user_model: str | None,
|
||||
route_type: str,
|
||||
) -> bool:
|
||||
"""Whether route_request would send this request somewhere, using only the models this worker has loaded.
|
||||
|
||||
A fully blocked model counts as not routable.
|
||||
"""
|
||||
try:
|
||||
return (
|
||||
_decide_dispatch(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type)
|
||||
is not _Dispatch.MODEL_NOT_FOUND
|
||||
)
|
||||
except litellm.PermissionDeniedError:
|
||||
return False
|
||||
|
||||
|
||||
async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited provider coroutines; the inferred union keeps route_request's callers typed
|
||||
data: dict, # mutable-ok: request body is the proxy-wide mutable dict contract shared with route_request
|
||||
llm_router: LitellmRouter | None,
|
||||
user_model: str | None,
|
||||
route_type: RouteType,
|
||||
router_only_settings_from_override: frozenset[str],
|
||||
user_api_key_dict: UserAPIKeyAuth | None = None,
|
||||
):
|
||||
raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=llm_router)
|
||||
|
||||
await add_shared_session_to_data(data)
|
||||
|
||||
raise_if_mock_testing_params_disallowed(data, allowed=mock_testing_params_allowed())
|
||||
|
||||
def direct_litellm_kwargs() -> Mapping[str, object]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in JSON_OBJECT_ADAPTER.validate_python(data).items()
|
||||
if key not in router_only_settings_from_override
|
||||
}
|
||||
)
|
||||
|
||||
# Preprocess Google GenAI generate content requests
|
||||
if route_type in ["agenerate_content", "agenerate_content_stream"]:
|
||||
# Map generationConfig to config parameter for Google GenAI compatibility
|
||||
if "generationConfig" in data and "config" not in data:
|
||||
data["config"] = data.pop("generationConfig")
|
||||
|
||||
match _decide_dispatch(data=data, llm_router=llm_router, user_model=user_model, route_type=route_type):
|
||||
case _Dispatch.ROUTER:
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
case _Dispatch.ROUTER_SPECIFIC_DEPLOYMENT:
|
||||
return getattr(llm_router, f"{route_type}")(**data, specific_deployment=True)
|
||||
case _Dispatch.ROUTER_THEN_LITELLM:
|
||||
try:
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
except Exception:
|
||||
# If router fails (e.g., model not found in router), fall back to direct call
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
case _Dispatch.LITELLM:
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
case _Dispatch.BATCH_COMPLETION if llm_router is not None:
|
||||
if data.get("fastest_response", False):
|
||||
return llm_router.abatch_completion_fastest_response(**data)
|
||||
models: Final = [model.strip() for model in data.pop("model").split(",")]
|
||||
return llm_router.abatch_completion(models=models, **data)
|
||||
case _Dispatch.USER_CONFIG:
|
||||
return _route_user_config_request(data, route_type)
|
||||
case _Dispatch.EVAL:
|
||||
# If a model is provided, get its credentials from the router
|
||||
model: Final = data.get("model")
|
||||
if model and llm_router:
|
||||
try:
|
||||
# Try to get deployment credentials for this model
|
||||
deployment_creds = llm_router.get_deployment_credentials(model_id=model)
|
||||
if not deployment_creds:
|
||||
# Try by model group name
|
||||
deployment: Final = llm_router.get_deployment_by_model_group_name(model_group_name=model)
|
||||
if (
|
||||
deployment
|
||||
and deployment.litellm_params
|
||||
and not llm_router.is_deployment_blocked(deployment)
|
||||
):
|
||||
deployment_creds = deployment.litellm_params.model_dump(exclude_none=True)
|
||||
|
||||
# If we found credentials, merge them into data (but don't override user-provided values)
|
||||
if deployment_creds:
|
||||
data.update(deployment_creds)
|
||||
except Exception:
|
||||
# If we can't get deployment creds, continue without them
|
||||
pass
|
||||
|
||||
return getattr(litellm, f"{route_type}")(**direct_litellm_kwargs())
|
||||
case _TeamModel(name=team_model_name):
|
||||
data["model"] = team_model_name
|
||||
return getattr(llm_router, f"{route_type}")(**data)
|
||||
case _Dispatch.MISSING_MODEL:
|
||||
raise ProxyMissingRequiredParamError(
|
||||
route=ROUTE_ENDPOINT_MAPPING.get(route_type, route_type),
|
||||
param="model",
|
||||
)
|
||||
case _Dispatch.A2A_AGENT:
|
||||
from litellm.proxy.agent_endpoints.a2a_routing import (
|
||||
route_a2a_agent_request,
|
||||
)
|
||||
|
||||
result: Final = await route_a2a_agent_request(data, route_type, user_api_key_dict=user_api_key_dict)
|
||||
if result is not None:
|
||||
return result
|
||||
case _Dispatch.BATCH_COMPLETION | _Dispatch.MODEL_NOT_FOUND:
|
||||
pass
|
||||
|
||||
_raise_model_not_found(data=data, route_type=route_type)
|
||||
|
|
|
|||
|
|
@ -2867,6 +2867,7 @@ class ProxyLogging:
|
|||
def has_pre_call_guardrails(
|
||||
self,
|
||||
request_metadata: Mapping[str, object],
|
||||
request_body: Mapping[str, object] | None = None,
|
||||
event_type: GuardrailEventHooks = GuardrailEventHooks.pre_call,
|
||||
) -> bool:
|
||||
"""
|
||||
|
|
@ -2875,17 +2876,19 @@ class ProxyLogging:
|
|||
Evaluated with the same predicate the pre-call loop uses, so a proxy configured only with
|
||||
post-call guardrails answers False. Callers that must pay a real cost to build the hook's
|
||||
input, such as streaming a batch input file off disk, use this to skip that work.
|
||||
``request_body`` supplies the request's top-level fields, such as ``stream``, that a
|
||||
guardrail scoped to streaming or non-streaming requests reads.
|
||||
|
||||
A content-enforcing ``CustomLogger`` counts too. It is not a guardrail and has no event
|
||||
hook to consult, but it judges the payload the same way, so a proxy configured only with
|
||||
one of those still has something to say about every record.
|
||||
"""
|
||||
if request_metadata.get("_guardrail_pipelines"):
|
||||
probe: Final = {**(request_body or {}), "metadata": dict(request_metadata)}
|
||||
if any(pipeline.mode == "pre_call" for _, pipeline in _policy_pipelines(probe)):
|
||||
return True
|
||||
caps: Final = ProxyLogging.callback_capabilities()
|
||||
if caps.has_content_enforcer:
|
||||
return True
|
||||
probe: Final = {"metadata": dict(request_metadata)}
|
||||
return any(
|
||||
isinstance(callback, CustomGuardrail) and callback.should_run_guardrail(data=probe, event_type=event_type)
|
||||
for callback in caps.resolved_callbacks
|
||||
|
|
@ -2901,6 +2904,7 @@ class ProxyLogging:
|
|||
guardrails_only: bool = False,
|
||||
skip_guardrails: bool = False,
|
||||
endpoint_type: EndpointType = EndpointType.GENERIC,
|
||||
before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload
|
||||
) -> None:
|
||||
pass
|
||||
|
||||
|
|
@ -2913,6 +2917,7 @@ class ProxyLogging:
|
|||
guardrails_only: bool = False,
|
||||
skip_guardrails: bool = False,
|
||||
endpoint_type: EndpointType = EndpointType.GENERIC,
|
||||
before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload
|
||||
) -> dict:
|
||||
pass
|
||||
|
||||
|
|
@ -2924,6 +2929,7 @@ class ProxyLogging:
|
|||
guardrails_only: bool = False,
|
||||
skip_guardrails: bool = False,
|
||||
endpoint_type: EndpointType = EndpointType.GENERIC,
|
||||
before_guardrails: Callable[[dict[str, object]], Awaitable[None]] | None = None, # mutable-ok: request payload
|
||||
) -> dict | None:
|
||||
"""
|
||||
Allows users to modify/reject the incoming request to the proxy, without having to deal with parsing Request body.
|
||||
|
|
@ -2936,6 +2942,10 @@ class ProxyLogging:
|
|||
With ``guardrails_only`` the walk is limited to guardrails and guardrail pipelines: rate
|
||||
limiting, budget accounting, prompt templates and hanging-request alerting are skipped.
|
||||
Use it to scan a payload that is not itself a request, such as one record of a batch file.
|
||||
|
||||
``before_guardrails`` is awaited once with the request, right before the first guardrail
|
||||
pipeline, guardrail or content enforcer that runs on it, or after the other callbacks when
|
||||
none does. An exception it raises ends the hook.
|
||||
"""
|
||||
verbose_proxy_logger.debug("Inside Proxy Logging Pre-call hook!")
|
||||
|
||||
|
|
@ -2986,9 +2996,16 @@ class ProxyLogging:
|
|||
raw_request_snapshot: Final[dict | None] = ( # mutable-ok: same request-payload shape as data
|
||||
independent_snapshot(data) if needs_raw_request_snapshot else None
|
||||
)
|
||||
pending_before_guardrails: Final = [before_guardrails] if before_guardrails is not None else []
|
||||
|
||||
async def reach_guardrails(request: dict[str, object]) -> None: # mutable-ok: request payload
|
||||
while pending_before_guardrails:
|
||||
await pending_before_guardrails.pop()(request)
|
||||
|
||||
try:
|
||||
if not skip_guardrails:
|
||||
if any(pipeline.mode == "pre_call" for _, pipeline in _policy_pipelines(data)):
|
||||
await reach_guardrails(data)
|
||||
data, _ = await self._maybe_execute_pipelines( # rebind-ok: pipeline edits feed the callback loop below
|
||||
data=data,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -3014,6 +3031,7 @@ class ProxyLogging:
|
|||
and (guardrails_only or not caps.has_pre_call_override)
|
||||
):
|
||||
if data is not None:
|
||||
await reach_guardrails(data)
|
||||
self._process_guardrail_metadata(data)
|
||||
return data
|
||||
|
||||
|
|
@ -3044,6 +3062,13 @@ class ProxyLogging:
|
|||
if getattr(_callback, "run_in_parallel", False):
|
||||
continue
|
||||
|
||||
if (
|
||||
pending_before_guardrails
|
||||
and _callback.should_run_guardrail(data=data, event_type=GuardrailEventHooks.pre_call)
|
||||
is True
|
||||
):
|
||||
await reach_guardrails(data)
|
||||
|
||||
data = await self._run_sequential_guardrail_callback(
|
||||
callback=_callback,
|
||||
data=data,
|
||||
|
|
@ -3062,6 +3087,9 @@ class ProxyLogging:
|
|||
if call_type in MCP_GUARDRAIL_CALL_TYPES and user_api_key_dict is None:
|
||||
continue
|
||||
|
||||
if _callback.enforces_request_content and data is not None:
|
||||
await reach_guardrails(data)
|
||||
|
||||
response: Exception | str | Mapping[str, object] | None = (
|
||||
await _callback.async_pre_call_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
|
|
@ -3105,6 +3133,9 @@ class ProxyLogging:
|
|||
end_time=end_time,
|
||||
)
|
||||
|
||||
if data is not None:
|
||||
await reach_guardrails(data)
|
||||
|
||||
if deferred_route_exc is not None and data is not None:
|
||||
data = await self._handle_sensitive_data_route_exception(deferred_route_exc, data, user_api_key_dict)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import contextvars
|
|||
import copy
|
||||
import datetime
|
||||
import json
|
||||
from collections.abc import AsyncIterable, Mapping
|
||||
from collections.abc import AsyncIterable, Awaitable, Mapping
|
||||
from types import MappingProxyType, SimpleNamespace
|
||||
from typing import AsyncGenerator, Callable, Final, Iterator, Literal, Optional, Sequence
|
||||
from urllib.parse import unquote_plus
|
||||
|
|
@ -33,6 +33,7 @@ from litellm.constants import (
|
|||
RETURN_RAW_MODEL_NAME_METADATA_KEY,
|
||||
STREAM_SSE_KEEPALIVE_PING_BYTES,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.opentelemetry import UserAPIKeyAuth
|
||||
from litellm.proxy.common_request_processing import (
|
||||
|
|
@ -80,6 +81,7 @@ from litellm.router import Router
|
|||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.utils import ModelResponseListIterator
|
||||
from litellm.router_utils.add_retry_fallback_headers import prepare_response_for_header_attachment
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
|
||||
def test_attach_guardrail_information_copies_recorded_entries_onto_model_response():
|
||||
|
|
@ -11901,6 +11903,610 @@ class TestStreamingContainerOwnershipRecordedBeforeDone:
|
|||
recorder.assert_awaited_once()
|
||||
|
||||
|
||||
class _CountingGuardrail(CustomGuardrail):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(guardrail_name="counting-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=True)
|
||||
self.models_seen: list[object] = []
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
self.models_seen.append(data.get("model"))
|
||||
return data
|
||||
|
||||
|
||||
class _CountingContentEnforcer(CustomLogger):
|
||||
enforces_request_content = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.calls = 0
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
self.calls += 1
|
||||
return data
|
||||
|
||||
|
||||
class _ModelRewritingHook(CustomLogger):
|
||||
def __init__(self, source_model: str = "virtual-model", target_model: str = "gpt-4o-mini") -> None:
|
||||
super().__init__()
|
||||
self.source_model = source_model
|
||||
self.target_model = target_model
|
||||
self.calls = 0
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
self.calls += 1
|
||||
if data.get("model") == self.source_model:
|
||||
data["model"] = self.target_model
|
||||
return data
|
||||
|
||||
|
||||
class _ModelReplacingHook(CustomLogger):
|
||||
def __init__(self, target_model: str) -> None:
|
||||
super().__init__()
|
||||
self.target_model = target_model
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
return {**data, "model": self.target_model}
|
||||
|
||||
|
||||
class _EmailMaskingHook(CustomLogger):
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
data["messages"] = [
|
||||
{**message, "content": message["content"].replace("jane@example.com", "[EMAIL]")}
|
||||
for message in data["messages"]
|
||||
]
|
||||
return data
|
||||
|
||||
|
||||
class _EmailMaskingGuardrail(CustomGuardrail):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(
|
||||
guardrail_name="email-masking-guardrail", event_hook=GuardrailEventHooks.pre_call, default_on=True
|
||||
)
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
for message in data["messages"]:
|
||||
message["content"] = message["content"].replace("jane@example.com", "[EMAIL]")
|
||||
return data
|
||||
|
||||
|
||||
class _ContentRecordingHook(CustomLogger):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.contents_seen: list[str] = []
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
self.contents_seen.extend(message["content"] for message in data["messages"])
|
||||
return data
|
||||
|
||||
|
||||
class _RejectingHook(CustomLogger):
|
||||
def __init__(self, model: str | None = None) -> None:
|
||||
super().__init__()
|
||||
self.model = model
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
if self.model in (None, data.get("model")):
|
||||
raise HTTPException(status_code=403, detail="rejected by hook")
|
||||
return data
|
||||
|
||||
|
||||
class _RateLimitedModelHook(CustomLogger):
|
||||
def __init__(self, model: str) -> None:
|
||||
super().__init__()
|
||||
self.model = model
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
|
||||
|
||||
if data.get("model") == self.model:
|
||||
raise ProxyRateLimitError(detail=f"rpm limit reached for {self.model}")
|
||||
return data
|
||||
|
||||
|
||||
class TestPreCallHooksForUnroutableModel:
|
||||
@staticmethod
|
||||
async def _run(
|
||||
monkeypatch,
|
||||
model: str,
|
||||
llm_router: Router | None,
|
||||
processor: ProxyBaseLLMRequestProcessing | None = None,
|
||||
skip_guardrails: bool = False,
|
||||
model_db_lookup: Callable[[str], Awaitable[bool]] | None = None,
|
||||
extra_body: Mapping[str, object] | None = None,
|
||||
) -> dict:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.common_utils import registry_read_through
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
|
||||
monkeypatch.setattr(
|
||||
registry_read_through,
|
||||
"model_registry_read_through",
|
||||
registry_read_through.RegistryReadThrough(
|
||||
resync=model_db_lookup or registry_read_through._resync_model_deployments,
|
||||
is_loaded=registry_read_through._model_is_loaded,
|
||||
),
|
||||
)
|
||||
target: Final = processor or ProxyBaseLLMRequestProcessing(data={})
|
||||
target.data.update(
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "email jane@example.com"}],
|
||||
**copy.deepcopy(dict(extra_body or {})),
|
||||
}
|
||||
)
|
||||
request: Final = Request(
|
||||
{"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""}
|
||||
)
|
||||
proxy_logging_obj: Final = ProxyLogging(user_api_key_cache=UserApiKeyCache())
|
||||
data, _ = await target.common_processing_pre_call_logic(
|
||||
request=request,
|
||||
general_settings={},
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(),
|
||||
proxy_logging_obj=proxy_logging_obj,
|
||||
proxy_config=proxy_server.ProxyConfig(),
|
||||
route_type="acompletion",
|
||||
llm_router=llm_router,
|
||||
skip_guardrails=skip_guardrails,
|
||||
)
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
def _router() -> Router:
|
||||
return Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}},
|
||||
{
|
||||
"model_name": "blocked-model",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"},
|
||||
"model_info": {"blocked": True},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_model_runs_only_the_hooks_before_the_first_guardrail(self, monkeypatch):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
enforcer: Final = _CountingContentEnforcer()
|
||||
hook: Final = _ModelRewritingHook()
|
||||
monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer])
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError) as exc_info:
|
||||
await self._run(monkeypatch, model="does-not-exist", llm_router=self._router())
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert hook.calls == 1
|
||||
assert guardrail.models_seen == []
|
||||
assert enforcer.calls == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hook_that_rewrites_unknown_model_to_routable_one_still_gets_guardrails(self, monkeypatch):
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
enforcer: Final = _CountingContentEnforcer()
|
||||
hook: Final = _ModelRewritingHook()
|
||||
monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer])
|
||||
|
||||
data: Final = await self._run(monkeypatch, model="virtual-model", llm_router=self._router())
|
||||
|
||||
assert data["model"] == "gpt-4o-mini"
|
||||
assert hook.calls == 1
|
||||
assert guardrail.models_seen == ["gpt-4o-mini"]
|
||||
assert enforcer.calls == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_model_never_reaches_a_hook_registered_after_a_guardrail(self, monkeypatch):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
guardrail: Final = _EmailMaskingGuardrail()
|
||||
recorder: Final = _ContentRecordingHook()
|
||||
rejecting_hook: Final = _RejectingHook()
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail, recorder, rejecting_hook])
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError) as exc_info:
|
||||
await self._run(monkeypatch, model="does-not-exist", llm_router=self._router())
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert recorder.contents_seen == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hook_registered_after_a_guardrail_sees_the_guardrail_output_for_a_renamed_model(self, monkeypatch):
|
||||
recorder: Final = _ContentRecordingHook()
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), _EmailMaskingGuardrail(), recorder])
|
||||
|
||||
data: Final = await self._run(monkeypatch, model="virtual-model", llm_router=self._router())
|
||||
|
||||
assert data["model"] == "gpt-4o-mini"
|
||||
assert recorder.contents_seen == ["email [EMAIL]"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routable_model_runs_every_hook_once(self, monkeypatch):
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
enforcer: Final = _CountingContentEnforcer()
|
||||
hook: Final = _ModelRewritingHook()
|
||||
monkeypatch.setattr(litellm, "callbacks", [hook, guardrail, enforcer])
|
||||
|
||||
await self._run(monkeypatch, model="gpt-4o-mini", llm_router=self._router())
|
||||
|
||||
assert (hook.calls, guardrail.models_seen, enforcer.calls) == (1, ["gpt-4o-mini"], 1)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_routable_model_keeps_the_configured_hook_order(self, monkeypatch):
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail, _RejectingHook()])
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
await self._run(monkeypatch, model="gpt-4o-mini", llm_router=self._router())
|
||||
|
||||
assert guardrail.models_seen == ["gpt-4o-mini"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fully_blocked_model_is_a_403_without_guardrails(self, monkeypatch):
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
||||
|
||||
with pytest.raises(litellm.PermissionDeniedError):
|
||||
await self._run(monkeypatch, model="blocked-model", llm_router=self._router())
|
||||
|
||||
assert guardrail.models_seen == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_hook_that_rewrites_blocked_model_to_routable_one_is_served(self, monkeypatch):
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(source_model="blocked-model"), guardrail])
|
||||
|
||||
data: Final = await self._run(monkeypatch, model="blocked-model", llm_router=self._router())
|
||||
|
||||
assert data["model"] == "gpt-4o-mini"
|
||||
assert guardrail.models_seen == ["gpt-4o-mini"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("callbacks", "expected"),
|
||||
[
|
||||
pytest.param(lambda: [_CountingGuardrail()], True, id="guardrail-applies"),
|
||||
pytest.param(lambda: [_CountingContentEnforcer()], True, id="content-enforcer"),
|
||||
pytest.param(lambda: [_ModelRewritingHook()], None, id="no-guardrail"),
|
||||
],
|
||||
)
|
||||
async def test_rejected_request_is_redacted_only_when_guardrails_were_bypassed(
|
||||
self, monkeypatch, callbacks, expected
|
||||
):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", callbacks())
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError):
|
||||
await self._run(
|
||||
monkeypatch,
|
||||
model="does-not-exist",
|
||||
llm_router=self._router(),
|
||||
processor=processor,
|
||||
)
|
||||
|
||||
logging_obj: Final = processor.data["litellm_logging_obj"]
|
||||
assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is expected
|
||||
assert logging_obj.model_call_details["standard_callback_dynamic_params"] is (
|
||||
logging_obj.standard_callback_dynamic_params
|
||||
)
|
||||
expected_content: Final = "redacted-by-litellm" if expected else "email jane@example.com"
|
||||
assert processor.data["messages"] == [{"role": "user", "content": expected_content}]
|
||||
assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_a_hook_rejects_before_guardrails_is_redacted(self, monkeypatch):
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RejectingHook(), guardrail])
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), processor=processor)
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
assert guardrail.models_seen == []
|
||||
assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}]
|
||||
assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"]
|
||||
logging_obj: Final = processor.data["litellm_logging_obj"]
|
||||
assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_rejected_after_the_routing_check_passed_is_not_redacted(self, monkeypatch):
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), guardrail, _RejectingHook()])
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor)
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
assert guardrail.models_seen == ["gpt-4o-mini"]
|
||||
assert processor.data["messages"] == [{"role": "user", "content": "email jane@example.com"}]
|
||||
logging_obj: Final = processor.data["litellm_logging_obj"]
|
||||
assert logging_obj.standard_callback_dynamic_params.get("turn_off_message_logging") is not True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_rejected_after_the_routing_check_passed_logs_the_request_as_sent(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), _CountingGuardrail(), _RejectingHook()])
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
|
||||
with pytest.raises(HTTPException):
|
||||
await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor)
|
||||
|
||||
snapshot: Final = processor.data["proxy_server_request"]["body"]
|
||||
assert processor.data["model"] == "gpt-4o-mini"
|
||||
assert (snapshot["model"], snapshot["messages"], "litellm_call_id" in snapshot) == (
|
||||
"virtual-model",
|
||||
[{"role": "user", "content": "email jane@example.com"}],
|
||||
False,
|
||||
), snapshot
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_request_a_hook_replaced_with_another_unknown_model_is_the_one_rejected_and_redacted(
|
||||
self, monkeypatch
|
||||
):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ModelReplacingHook("still-unknown"), _CountingGuardrail()])
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError):
|
||||
await self._run(monkeypatch, model="virtual-model", llm_router=self._router(), processor=processor)
|
||||
|
||||
assert processor.data["model"] == "still-unknown"
|
||||
assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("extra_body", "expected_body"),
|
||||
[
|
||||
pytest.param(
|
||||
{"system": [{"type": "text", "text": "email jane@example.com"}]},
|
||||
{"system": "redacted-by-litellm"},
|
||||
id="anthropic-system",
|
||||
),
|
||||
pytest.param(
|
||||
{"instructions": "email jane@example.com"},
|
||||
{"instructions": "redacted-by-litellm"},
|
||||
id="responses-instructions",
|
||||
),
|
||||
pytest.param(
|
||||
{"contents": [{"role": "user", "parts": [{"text": "email jane@example.com"}]}]},
|
||||
{"contents": [{"role": "user", "parts": [{"text": "redacted-by-litellm"}]}]},
|
||||
id="gemini-contents",
|
||||
),
|
||||
pytest.param(
|
||||
{"systemInstruction": {"parts": [{"text": "email jane@example.com"}]}},
|
||||
{"systemInstruction": {"parts": [{"text": "redacted-by-litellm"}]}},
|
||||
id="gemini-system-instruction",
|
||||
),
|
||||
pytest.param(
|
||||
{"config": {"system_instruction": {"parts": [{"text": "email jane@example.com"}]}, "temperature": 0}},
|
||||
{"config": {"system_instruction": {"parts": [{"text": "redacted-by-litellm"}]}, "temperature": 0}},
|
||||
id="gemini-config-system-instruction",
|
||||
),
|
||||
pytest.param(
|
||||
{"prompt": "email jane@example.com", "input": ["email jane@example.com"], "max_tokens": 5},
|
||||
{"prompt": "", "input": "", "max_tokens": 5},
|
||||
id="prompt-and-input",
|
||||
),
|
||||
pytest.param(
|
||||
{"tools": [{"type": "function", "function": {"name": "mail", "description": "jane@example.com"}}]},
|
||||
{"tools": []},
|
||||
id="tool-definitions",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_rejected_request_redacts_every_guardrail_scanned_field(self, monkeypatch, extra_body, expected_body):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", [_CountingGuardrail()])
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError):
|
||||
await self._run(
|
||||
monkeypatch,
|
||||
model="does-not-exist",
|
||||
llm_router=self._router(),
|
||||
processor=processor,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
|
||||
snapshot: Final = processor.data["proxy_server_request"]["body"]
|
||||
assert {key: processor.data[key] for key in expected_body} == expected_body
|
||||
assert {key: snapshot[key] for key in expected_body} == expected_body
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_model_hooks_left_unchanged_gets_one_registry_read_through(self, monkeypatch):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", [_CountingGuardrail()])
|
||||
looked_up: Final = list[str]()
|
||||
|
||||
async def no_db_row(model_name: str) -> bool:
|
||||
looked_up.append(model_name)
|
||||
return False
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError):
|
||||
await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), model_db_lookup=no_db_row)
|
||||
|
||||
assert looked_up == ["does-not-exist"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_rewritten_by_a_hook_is_looked_up_once_under_its_new_name(self, monkeypatch):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"callbacks",
|
||||
[_ModelRewritingHook(source_model="unknown-virtual", target_model="db-only-model")],
|
||||
)
|
||||
looked_up: Final = list[str]()
|
||||
|
||||
async def no_db_row(model_name: str) -> bool:
|
||||
looked_up.append(model_name)
|
||||
return False
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError):
|
||||
await self._run(monkeypatch, model="unknown-virtual", llm_router=self._router(), model_db_lookup=no_db_row)
|
||||
|
||||
assert looked_up == ["db-only-model"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_virtual_model_a_hook_maps_to_a_loaded_model_needs_no_db_lookup(self, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook()])
|
||||
looked_up: Final = list[str]()
|
||||
|
||||
async def no_db_row(model_name: str) -> bool:
|
||||
looked_up.append(model_name)
|
||||
return False
|
||||
|
||||
data: Final = await self._run(
|
||||
monkeypatch, model="virtual-model", llm_router=self._router(), model_db_lookup=no_db_row
|
||||
)
|
||||
|
||||
assert data["model"] == "gpt-4o-mini"
|
||||
assert looked_up == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unknown_model_without_guardrails_logs_the_request_the_hooks_produced(self, monkeypatch):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
monkeypatch.setattr(litellm, "callbacks", [_EmailMaskingHook()])
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError):
|
||||
await self._run(monkeypatch, model="does-not-exist", llm_router=self._router(), processor=processor)
|
||||
|
||||
snapshot: Final = processor.data["proxy_server_request"]["body"]
|
||||
assert snapshot["messages"] == [{"role": "user", "content": "email [EMAIL]"}]
|
||||
assert snapshot["litellm_call_id"] == processor.data["litellm_call_id"]
|
||||
assert snapshot["metadata"] is processor.data["metadata"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("stream_scope", "stream", "expected_content"),
|
||||
[
|
||||
pytest.param("streaming", True, "redacted-by-litellm", id="streaming-guardrail-streaming-request"),
|
||||
pytest.param("streaming", False, "email jane@example.com", id="streaming-guardrail-plain-request"),
|
||||
pytest.param("non_streaming", True, "email jane@example.com", id="plain-guardrail-streaming-request"),
|
||||
],
|
||||
)
|
||||
async def test_rejected_request_is_redacted_when_a_guardrail_scoped_to_its_stream_mode_applies(
|
||||
self, monkeypatch, stream_scope, stream, expected_content
|
||||
):
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
guardrail.apply_stream_scope(stream_scope)
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(data={})
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError):
|
||||
await self._run(
|
||||
monkeypatch,
|
||||
model="does-not-exist",
|
||||
llm_router=self._router(),
|
||||
processor=processor,
|
||||
extra_body={"stream": stream},
|
||||
)
|
||||
|
||||
assert processor.data["messages"] == [{"role": "user", "content": expected_content}]
|
||||
assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_caller_that_skips_guardrails_runs_content_enforcers_and_no_guardrail_for_a_renamed_model(
|
||||
self, monkeypatch
|
||||
):
|
||||
enforcer: Final = _CountingContentEnforcer()
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [_ModelRewritingHook(), enforcer, guardrail])
|
||||
|
||||
data: Final = await self._run(
|
||||
monkeypatch, model="virtual-model", llm_router=self._router(), skip_guardrails=True
|
||||
)
|
||||
|
||||
assert data["model"] == "gpt-4o-mini"
|
||||
assert enforcer.calls == 1
|
||||
assert guardrail.models_seen == []
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("primary", "fallback", "rejecting_hook", "expected_status"),
|
||||
[
|
||||
pytest.param("gpt-4o-mini", "does-not-exist", "unused-model", 400, id="fallback-model-unknown"),
|
||||
pytest.param("does-not-exist", "gpt-4o-mini", "gpt-4o-mini", 403, id="primary-model-unknown"),
|
||||
],
|
||||
)
|
||||
async def test_rate_limit_fallback_logs_the_attempt_rejected_before_its_guardrails(
|
||||
self, monkeypatch, primary, fallback, rejecting_hook, expected_status
|
||||
):
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.common_utils import registry_read_through
|
||||
from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
|
||||
monkeypatch.setattr(
|
||||
registry_read_through,
|
||||
"model_registry_read_through",
|
||||
registry_read_through.RegistryReadThrough(
|
||||
resync=registry_read_through._resync_model_deployments,
|
||||
is_loaded=registry_read_through._model_is_loaded,
|
||||
),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
litellm,
|
||||
"callbacks",
|
||||
[_RateLimitedModelHook(model=primary), _RejectingHook(model=rejecting_hook), _CountingGuardrail()],
|
||||
)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{"model_name": "gpt-4o-mini", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}
|
||||
],
|
||||
fallbacks=[{primary: [fallback]}],
|
||||
)
|
||||
processor: Final = ProxyBaseLLMRequestProcessing(
|
||||
data={"model": primary, "messages": [{"role": "user", "content": "email jane@example.com"}]}
|
||||
)
|
||||
|
||||
request: Final = Request(
|
||||
{"type": "http", "method": "POST", "path": "/v1/chat/completions", "headers": [], "query_string": b""}
|
||||
)
|
||||
|
||||
with pytest.raises((ProxyModelNotFoundError, HTTPException)) as exc_info:
|
||||
await processor._pre_call_with_fallbacks(
|
||||
request=request,
|
||||
general_settings={},
|
||||
proxy_logging_obj=ProxyLogging(user_api_key_cache=UserApiKeyCache()),
|
||||
user_api_key_dict=ProxyUserAPIKeyAuth(),
|
||||
version=None,
|
||||
proxy_config=proxy_server.ProxyConfig(),
|
||||
user_model=None,
|
||||
user_temperature=None,
|
||||
user_request_timeout=None,
|
||||
user_max_tokens=None,
|
||||
user_api_base=None,
|
||||
model=None,
|
||||
route_type="acompletion",
|
||||
llm_router=router,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == expected_status
|
||||
assert processor.data["model"] == "does-not-exist"
|
||||
assert processor.data["messages"] == [{"role": "user", "content": "redacted-by-litellm"}]
|
||||
assert processor.data["proxy_server_request"]["body"]["messages"] == processor.data["messages"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_router_skips_the_routability_check(self, monkeypatch):
|
||||
guardrail: Final = _CountingGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [guardrail])
|
||||
|
||||
data: Final = await self._run(monkeypatch, model="router-alias-only-known-at-dispatch", llm_router=None)
|
||||
|
||||
assert data["model"] == "router-alias-only-known-at-dispatch"
|
||||
assert guardrail.models_seen == ["router-alias-only-known-at-dispatch"]
|
||||
|
||||
|
||||
class TestResponseCostFromLoggingObjForBridgedMessages:
|
||||
"""A /v1/messages request answered through the chat bridge arrives here as an Anthropic body whose
|
||||
usage has no reasoning field, while the cost it recomputes must still price the reasoning tokens
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.route_llm_request import (
|
||||
JSON_OBJECT_ADAPTER,
|
||||
MockTestingParamsDisabledError,
|
||||
ProxyMissingRequiredParamError,
|
||||
ProxyModelNotFoundError,
|
||||
UserConfigRouters,
|
||||
route_request,
|
||||
|
|
@ -2186,6 +2188,184 @@ def test_proxy_model_not_found_error_keeps_the_raw_model_only_in_the_client_resp
|
|||
assert error.spend_log_error_message.startswith("/chat/completions: Invalid model name passed in")
|
||||
|
||||
|
||||
def _router_with_gpt4o_mini(router_general_settings: RouterGeneralSettings | None = None):
|
||||
import litellm
|
||||
|
||||
return litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "gpt-4o-mini",
|
||||
"litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"},
|
||||
}
|
||||
],
|
||||
router_general_settings=router_general_settings,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"extra_body",
|
||||
[
|
||||
pytest.param({}, id="plain"),
|
||||
pytest.param({"user_config": None}, id="null-user-config"),
|
||||
pytest.param({"router_settings_override": {"num_retries": 3}}, id="key-router-settings"),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_raise_if_model_not_routable_raises_for_unknown_model(extra_body):
|
||||
from litellm.proxy.route_llm_request import raise_if_model_not_routable
|
||||
|
||||
with pytest.raises(ProxyModelNotFoundError) as exc_info:
|
||||
await raise_if_model_not_routable(
|
||||
data={"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}], **extra_body},
|
||||
llm_router=_router_with_gpt4o_mini(),
|
||||
user_model=None,
|
||||
route_type="acompletion",
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 400
|
||||
assert "does-not-exist" in str(exc_info.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("data", "route_type", "expected_error"),
|
||||
[
|
||||
pytest.param(
|
||||
{"model": "does-not-exist"},
|
||||
"atext_completion",
|
||||
ProxyMissingRequiredParamError,
|
||||
id="missing-prompt",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}], "mock_timeout": True},
|
||||
"acompletion",
|
||||
MockTestingParamsDisabledError,
|
||||
id="mock-params-disallowed",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_raise_if_model_not_routable_reports_body_errors_before_the_model(data, route_type, expected_error):
|
||||
from litellm.proxy.route_llm_request import raise_if_model_not_routable
|
||||
|
||||
with pytest.raises(expected_error):
|
||||
await raise_if_model_not_routable(
|
||||
data=data,
|
||||
llm_router=_router_with_gpt4o_mini(),
|
||||
user_model=None,
|
||||
route_type=route_type,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raise_if_model_not_routable_reports_a_malformed_router_settings_override_like_route_request():
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.route_llm_request import raise_if_model_not_routable
|
||||
|
||||
data: Final = {
|
||||
"model": "does-not-exist",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"router_settings_override": "not-an-object",
|
||||
}
|
||||
|
||||
with pytest.raises(ProxyException) as from_check:
|
||||
await raise_if_model_not_routable(
|
||||
data=data, llm_router=_router_with_gpt4o_mini(), user_model=None, route_type="acompletion"
|
||||
)
|
||||
with pytest.raises(ProxyException) as from_route:
|
||||
await route_request(dict(data), _router_with_gpt4o_mini(), None, "acompletion")
|
||||
|
||||
assert from_check.value.param == from_route.value.param == "router_settings_override"
|
||||
assert from_check.value.message == from_route.value.message
|
||||
assert data["router_settings_override"] == "not-an-object"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raise_if_model_not_routable_leaves_the_router_settings_override_for_route_request():
|
||||
from litellm.proxy.route_llm_request import raise_if_model_not_routable
|
||||
|
||||
data: Final = {
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"router_settings_override": {"num_retries": 3},
|
||||
}
|
||||
|
||||
await raise_if_model_not_routable(
|
||||
data=data, llm_router=_router_with_gpt4o_mini(), user_model=None, route_type="acompletion"
|
||||
)
|
||||
|
||||
assert data == {
|
||||
"model": "gpt-4o-mini",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"router_settings_override": {"num_retries": 3},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data, route_type, router_mutator",
|
||||
[
|
||||
(
|
||||
{"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]},
|
||||
"acompletion",
|
||||
None,
|
||||
),
|
||||
(
|
||||
{"model": "openai/anything", "messages": [{"role": "user", "content": "hi"}]},
|
||||
"acompletion",
|
||||
"wildcard",
|
||||
),
|
||||
(
|
||||
{"model": "does-not-exist", "api_key": "sk-user", "messages": [{"role": "user", "content": "hi"}]},
|
||||
"acompletion",
|
||||
None,
|
||||
),
|
||||
(
|
||||
{"model": "does-not-exist,gpt-4o-mini", "messages": [{"role": "user", "content": "hi"}]},
|
||||
"acompletion",
|
||||
None,
|
||||
),
|
||||
(
|
||||
{"model": "does-not-exist", "messages": [{"role": "user", "content": "hi"}]},
|
||||
"acompletion",
|
||||
"pass_through_all_models",
|
||||
),
|
||||
(
|
||||
{"model": "does-not-exist", "input": "hello"},
|
||||
"amoderation",
|
||||
None,
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_raise_if_model_not_routable_allows_routable_requests(data, route_type, router_mutator):
|
||||
from litellm.proxy.route_llm_request import raise_if_model_not_routable
|
||||
|
||||
if router_mutator == "wildcard":
|
||||
import litellm
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "openai/*",
|
||||
"litellm_params": {"model": "openai/*", "api_key": "sk-test"},
|
||||
}
|
||||
]
|
||||
)
|
||||
elif router_mutator == "pass_through_all_models":
|
||||
router = _router_with_gpt4o_mini(router_general_settings=RouterGeneralSettings(pass_through_all_models=True))
|
||||
else:
|
||||
router = _router_with_gpt4o_mini()
|
||||
|
||||
assert (
|
||||
await raise_if_model_not_routable(
|
||||
data=data,
|
||||
llm_router=router,
|
||||
user_model=None,
|
||||
route_type=route_type,
|
||||
)
|
||||
is None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_route_request_without_model_on_model_routed_endpoint_is_a_400():
|
||||
import litellm
|
||||
|
|
@ -2370,6 +2550,72 @@ async def test_null_user_config_is_treated_as_absent() -> None:
|
|||
assert response.choices[0].message.content == "from the main router"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_route_request_without_a_model_sends_to_the_team_deployment_by_its_public_name():
|
||||
import litellm
|
||||
|
||||
router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "internal-team-gpt",
|
||||
"litellm_params": {"model": "openai/gpt-4o", "api_key": "fake", "mock_response": "from-team"},
|
||||
"model_info": {"id": "team-gpt-1", "team_id": "team-a", "team_public_model_name": "team-gpt"},
|
||||
}
|
||||
]
|
||||
)
|
||||
data = {
|
||||
"model": None,
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"metadata": {"user_api_key_team_id": "team-a"},
|
||||
}
|
||||
|
||||
llm_call = await route_request(data=data, llm_router=router, user_model=None, route_type="acompletion")
|
||||
response = await llm_call
|
||||
|
||||
assert response.choices[0].message.content == "from-team"
|
||||
assert data["model"] == "team-gpt"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_route_request_read_through_sends_through_the_router_it_built(monkeypatch):
|
||||
"""With no router loaded yet, a model the read-through finds is sent through the router the read-through built."""
|
||||
import litellm
|
||||
from litellm.proxy import proxy_server
|
||||
|
||||
model_name = "e2e-first-db-model"
|
||||
built_router = litellm.Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": model_name,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o",
|
||||
"api_key": "fake",
|
||||
"mock_response": "hello-from-new-router",
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
fake_prisma, _ = _fake_prisma_client_with_models([_db_model_row(model_name, "unused")])
|
||||
monkeypatch.setattr(proxy_server, "prisma_client", fake_prisma)
|
||||
monkeypatch.setattr(proxy_server, "store_model_in_db", True)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
|
||||
async def build_router_from_db(prisma_client, proxy_logging_obj):
|
||||
monkeypatch.setattr(proxy_server, "llm_router", built_router)
|
||||
|
||||
monkeypatch.setattr(proxy_server.proxy_config, "add_deployment", build_router_from_db)
|
||||
|
||||
llm_call = await route_request(
|
||||
data={"model": model_name, "messages": [{"role": "user", "content": "hi"}]},
|
||||
llm_router=None,
|
||||
user_model=None,
|
||||
route_type="acompletion",
|
||||
)
|
||||
response = await llm_call
|
||||
|
||||
assert response.choices[0].message.content == "hello-from-new-router"
|
||||
|
||||
|
||||
def _registered_bedrock_kb_search(team_id: str) -> dict[str, object]:
|
||||
"""The request a team key's /v1/vector_stores/{id}/search carries once the registry row is merged in."""
|
||||
return {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict
|
||||
from typing import Any, Dict, Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -266,10 +266,14 @@ def test_has_pre_call_guardrails_is_false_without_callbacks(proxy_logging, monke
|
|||
assert proxy_logging.has_pre_call_guardrails({}) is False
|
||||
|
||||
|
||||
def test_has_pre_call_guardrails_is_true_for_a_configured_pipeline(proxy_logging, monkeypatch):
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
@pytest.mark.parametrize(("mode", "expected"), [("pre_call", True), ("post_call", False)])
|
||||
def test_has_pre_call_guardrails_follows_the_pipeline_mode(proxy_logging, monkeypatch, mode, expected):
|
||||
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep
|
||||
|
||||
assert proxy_logging.has_pre_call_guardrails({"_guardrail_pipelines": ["p1"]}) is True
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
pipeline: Final = GuardrailPipeline(mode=mode, steps=[PipelineStep(guardrail="g")])
|
||||
|
||||
assert proxy_logging.has_pre_call_guardrails({"_guardrail_pipelines": [("policy", pipeline)]}) is expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -995,3 +999,134 @@ async def test_guardrails_only_and_skip_guardrails_are_mutually_exclusive(
|
|||
guardrails_only=True,
|
||||
skip_guardrails=True,
|
||||
)
|
||||
|
||||
|
||||
class _NamedHook(CustomLogger):
|
||||
def __init__(self, name: str, seen: list[str]) -> None:
|
||||
super().__init__()
|
||||
self.name = name
|
||||
self.seen = seen
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
self.seen.append(self.name)
|
||||
return data
|
||||
|
||||
|
||||
class _NamedEnforcer(_NamedHook):
|
||||
enforces_request_content = True
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
self.seen.append(self.name)
|
||||
return data
|
||||
|
||||
|
||||
class _NamedGuardrail(CustomGuardrail):
|
||||
def __init__(self, name: str, seen: list[str], default_on: bool = True, run_in_parallel: bool = False) -> None:
|
||||
super().__init__(guardrail_name=name, event_hook=GuardrailEventHooks.pre_call, default_on=default_on)
|
||||
self.seen = seen
|
||||
self.run_in_parallel = run_in_parallel
|
||||
|
||||
async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type):
|
||||
self.seen.append(self.guardrail_name)
|
||||
return data
|
||||
|
||||
|
||||
def _callbacks(spec: str, seen: list[str]) -> list[CustomLogger]:
|
||||
def build(name: str) -> CustomLogger:
|
||||
if name.startswith("enforcer"):
|
||||
return _NamedEnforcer(name, seen)
|
||||
if name.startswith("off-guardrail"):
|
||||
return _NamedGuardrail(name, seen, default_on=False)
|
||||
if name.startswith("parallel-guardrail"):
|
||||
return _NamedGuardrail(name, seen, run_in_parallel=True)
|
||||
if name.startswith("guardrail"):
|
||||
return _NamedGuardrail(name, seen)
|
||||
return _NamedHook(name, seen)
|
||||
|
||||
return [build(name) for name in spec.split()]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("callbacks", "expected"),
|
||||
[
|
||||
pytest.param("hook-a guardrail hook-b", ["hook-a", "check", "guardrail", "hook-b"], id="guardrail"),
|
||||
pytest.param("hook-a enforcer hook-b", ["hook-a", "check", "enforcer", "hook-b"], id="content-enforcer"),
|
||||
pytest.param(
|
||||
"off-guardrail hook-a guardrail hook-b",
|
||||
["hook-a", "check", "guardrail", "hook-b"],
|
||||
id="guardrail-that-does-not-apply",
|
||||
),
|
||||
pytest.param(
|
||||
"hook-a parallel-guardrail hook-b",
|
||||
["hook-a", "hook-b", "check", "parallel-guardrail"],
|
||||
id="parallel-guardrail",
|
||||
),
|
||||
pytest.param("hook-a hook-b", ["hook-a", "hook-b", "check"], id="no-guardrail"),
|
||||
pytest.param("", ["check"], id="no-callbacks"),
|
||||
],
|
||||
)
|
||||
async def test_before_guardrails_runs_once_right_before_the_first_guardrail(
|
||||
proxy_logging, make_user_api_key_auth, monkeypatch, callbacks, expected
|
||||
):
|
||||
seen: Final = list[str]()
|
||||
monkeypatch.setattr(litellm, "callbacks", _callbacks(callbacks, seen))
|
||||
proxy_logging.slack_alerting_instance = MagicMock(alerting=None)
|
||||
|
||||
async def check(request: dict) -> None:
|
||||
seen.append("check")
|
||||
|
||||
await proxy_logging.pre_call_hook(
|
||||
user_api_key_dict=make_user_api_key_auth(),
|
||||
data={"model": "m", "messages": [{"role": "user", "content": "hi"}]},
|
||||
call_type="acompletion",
|
||||
before_guardrails=check,
|
||||
)
|
||||
|
||||
assert seen == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_before_guardrails_runs_before_a_guardrail_pipeline(proxy_logging, make_user_api_key_auth, monkeypatch):
|
||||
from litellm.types.proxy.policy_engine.pipeline_types import GuardrailPipeline, PipelineStep
|
||||
|
||||
seen: Final = list[str]()
|
||||
monkeypatch.setattr(litellm, "callbacks", _callbacks("hook-a guardrail-in-pipeline", seen))
|
||||
proxy_logging.slack_alerting_instance = MagicMock(alerting=None)
|
||||
pipeline: Final = GuardrailPipeline(mode="pre_call", steps=[PipelineStep(guardrail="guardrail-in-pipeline")])
|
||||
|
||||
async def check(request: dict) -> None:
|
||||
seen.append("check")
|
||||
|
||||
await proxy_logging.pre_call_hook(
|
||||
user_api_key_dict=make_user_api_key_auth(),
|
||||
data={
|
||||
"model": "m",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"metadata": {"_guardrail_pipelines": [("policy", pipeline)]},
|
||||
},
|
||||
call_type="acompletion",
|
||||
before_guardrails=check,
|
||||
)
|
||||
|
||||
assert seen == ["check", "guardrail-in-pipeline", "hook-a"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failing_before_guardrails_stops_the_walk(proxy_logging, make_user_api_key_auth, monkeypatch):
|
||||
seen: Final = list[str]()
|
||||
monkeypatch.setattr(litellm, "callbacks", _callbacks("hook-a guardrail hook-b", seen))
|
||||
proxy_logging.slack_alerting_instance = MagicMock(alerting=None)
|
||||
|
||||
async def reject(request: dict) -> None:
|
||||
raise HTTPException(status_code=400, detail="model cannot route")
|
||||
|
||||
with pytest.raises(HTTPException, match="model cannot route"):
|
||||
await proxy_logging.pre_call_hook(
|
||||
user_api_key_dict=make_user_api_key_auth(),
|
||||
data={"model": "m", "messages": [{"role": "user", "content": "hi"}]},
|
||||
call_type="acompletion",
|
||||
before_guardrails=reject,
|
||||
)
|
||||
|
||||
assert seen == ["hook-a"]
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue