mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge 67e5637f63 into aa7917a3d2
This commit is contained in:
commit
1355afb21d
27 changed files with 3529 additions and 14 deletions
|
|
@ -123,6 +123,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
|
|||
# Realtime / streaming
|
||||
"/v1/realtime",
|
||||
"/realtime",
|
||||
# ORACLE router feedback and state: the program bindings live in the serving process
|
||||
"/oracle_router/",
|
||||
"/v1/live/sessions",
|
||||
"/live/sessions",
|
||||
# Health & ops
|
||||
|
|
|
|||
51
litellm/proxy/example_config_yaml/oracle_router_example.yaml
Normal file
51
litellm/proxy/example_config_yaml/oracle_router_example.yaml
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
# ORACLE router: one model per agentic program, learned from verified outcomes.
|
||||
#
|
||||
# Clients call `model: oracle-router` and tag every request of a task with the same program id:
|
||||
# POST /v1/chat/completions { "model": "oracle-router", "metadata": { "program_id": "task-17" }, ... }
|
||||
# The first request binds the program to a model; later requests reuse it. On the last request add
|
||||
# "metadata": { "program_id": "task-17", "program_done": true, "program_score": 1.0 }
|
||||
# or POST /oracle_router/feedback { "program_id": "task-17", "score": 1.0 } once the task is graded.
|
||||
#
|
||||
# Required env vars: OPENAI_API_KEY, LITELLM_MASTER_KEY.
|
||||
|
||||
model_list:
|
||||
- model_name: oracle-router
|
||||
litellm_params:
|
||||
model: auto_router/oracle_router
|
||||
oracle_router_config:
|
||||
available_models: ["smart", "fast"] # strongest first
|
||||
decision_maker:
|
||||
type: thompson # fixed | pre_routing | custom
|
||||
weights: { quality: 0.7, cost: 0.3 } # the adaptive router's quality/cost trade-off
|
||||
verifier:
|
||||
type: guardrail # reported | custom
|
||||
guardrail: answer-judge
|
||||
|
||||
# Let LiteLLM's complexity router propose the model and have ORACLE bind, verify and report it:
|
||||
# decision_maker: { type: pre_routing, router: tiered-router }
|
||||
|
||||
- model_name: smart
|
||||
litellm_params:
|
||||
model: openai/gpt-4o
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
- model_name: fast
|
||||
litellm_params:
|
||||
model: openai/gpt-4o-mini
|
||||
api_key: os.environ/OPENAI_API_KEY
|
||||
|
||||
guardrails:
|
||||
- guardrail_name: answer-judge
|
||||
litellm_params:
|
||||
guardrail: llm_as_a_judge
|
||||
mode: post_call
|
||||
default_on: false
|
||||
judge_model: smart
|
||||
overall_threshold: 50
|
||||
criteria:
|
||||
- name: task_completed
|
||||
weight: 100
|
||||
description: The assistant completed the user's task correctly, judged from the whole conversation.
|
||||
|
||||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
|
|
@ -3970,7 +3970,7 @@ async def clear_cache() -> ReconcileOutcome:
|
|||
config_models: Final = []
|
||||
db_model_ids: Final = []
|
||||
|
||||
db_router_names: Final = set()
|
||||
db_router_names: Final[set[str]] = set() # mutable-ok: filled in the classification pass below
|
||||
|
||||
for model in current_models:
|
||||
model_info = model.get("model_info", {})
|
||||
|
|
@ -3979,7 +3979,7 @@ async def clear_cache() -> ReconcileOutcome:
|
|||
# Auto-router deployments (and only those) are wiped here, in the
|
||||
# same pass, so the reload rebuilds them -- see the comment below.
|
||||
model_name = model.get("model_name")
|
||||
if model_name is not None and str(model.get("litellm_params", {}).get("model", "")).startswith(
|
||||
if isinstance(model_name, str) and str(model.get("litellm_params", {}).get("model", "")).startswith(
|
||||
"auto_router/"
|
||||
):
|
||||
db_router_names.add(model_name)
|
||||
|
|
@ -4028,6 +4028,7 @@ async def clear_cache() -> ReconcileOutcome:
|
|||
llm_router.registry().plugins("complexity").pop(model_name, None)
|
||||
llm_router.registry().plugins("adaptive").pop(model_name, None)
|
||||
llm_router.registry().plugins("quality").pop(model_name, None)
|
||||
llm_router.registry().plugins("oracle").pop(model_name, None)
|
||||
|
||||
# Reload only DB models. _add_deployment_locked, not add_deployment: this
|
||||
# coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not
|
||||
|
|
|
|||
117
litellm/proxy/management_endpoints/oracle_router_endpoints.py
Normal file
117
litellm/proxy/management_endpoints/oracle_router_endpoints.py
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
"""
|
||||
ORACLE ROUTER MANAGEMENT ENDPOINTS
|
||||
|
||||
POST /oracle_router/feedback - Report a finished program so its verifier runs and the router learns
|
||||
GET /oracle_router/state - Live decision-maker state, bindings and recent feedback per router
|
||||
"""
|
||||
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Annotated, Final
|
||||
|
||||
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.router_strategy.oracle_router.oracle_router import VerificationBacklogFull
|
||||
from litellm.types.management_endpoints.oracle_router_endpoints import (
|
||||
OracleRouterFeedbackRequest,
|
||||
OracleRouterFeedbackResponse,
|
||||
OracleRouterStateResponse,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
|
||||
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, ProgramBinding
|
||||
else:
|
||||
try:
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
||||
def _is_admin(user_api_key_dict: UserAPIKeyAuth) -> bool:
|
||||
return user_api_key_dict.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
|
||||
|
||||
|
||||
def _oracle_routers() -> tuple["OracleRouter", ...]:
|
||||
from litellm.proxy.proxy_server import llm_router
|
||||
|
||||
registry: Final = llm_router.registry().plugins("oracle") if llm_router is not None else {}
|
||||
tagged_entries: Final = tuple(chain.from_iterable(registry.values()))
|
||||
return tuple(tagged.strategy for tagged in tagged_entries)
|
||||
|
||||
|
||||
def _find_program(
|
||||
routers: tuple["OracleRouter", ...], program_id: str, user_api_key_dict: UserAPIKeyAuth
|
||||
) -> tuple["OracleRouter", "ProgramBinding"] | None:
|
||||
"""The caller's program with this id. An admin may also reach another key's program of that id."""
|
||||
for candidate in routers:
|
||||
owned = candidate.binding(program_id, user_api_key_dict.api_key)
|
||||
if owned is not None:
|
||||
return candidate, owned
|
||||
if not _is_admin(user_api_key_dict):
|
||||
return None
|
||||
for candidate in routers:
|
||||
for any_owner in candidate.bindings_named(program_id):
|
||||
return candidate, any_owner
|
||||
return None
|
||||
|
||||
|
||||
@router.post(
|
||||
"/oracle_router/feedback",
|
||||
tags=["oracle_router"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=OracleRouterFeedbackResponse,
|
||||
)
|
||||
async def submit_oracle_router_feedback(
|
||||
data: OracleRouterFeedbackRequest,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> OracleRouterFeedbackResponse:
|
||||
"""Complete a program: its model slot is released now, its verifier runs in the background.
|
||||
|
||||
Programs are private to the key that started them; an admin may complete any key's program. 404 when
|
||||
the caller holds no program with that id, which also covers programs completed earlier. A program none
|
||||
of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed
|
||||
response completes it with this feedback.
|
||||
"""
|
||||
routers: Final = _oracle_routers()
|
||||
if not routers:
|
||||
raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."})
|
||||
found: Final = _find_program(routers, data.program_id, user_api_key_dict)
|
||||
if found is None:
|
||||
raise HTTPException(
|
||||
status_code=404, detail={"error": f"Unknown or already completed program {data.program_id!r}."}
|
||||
)
|
||||
holder, binding = found
|
||||
try:
|
||||
holder.complete(
|
||||
data.program_id, score=data.score, cost=data.cost, payload=data.payload, owner=binding.api_key_hash
|
||||
)
|
||||
except VerificationBacklogFull as error:
|
||||
raise HTTPException(status_code=503, detail={"error": str(error)}) from error
|
||||
return OracleRouterFeedbackResponse(
|
||||
program_id=data.program_id,
|
||||
router_name=holder.router_name,
|
||||
model=binding.model,
|
||||
pending_verifications=holder.pending_verifications,
|
||||
deferred=binding.requests == 0,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/oracle_router/state",
|
||||
tags=["oracle_router"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
response_model=OracleRouterStateResponse,
|
||||
)
|
||||
async def get_oracle_router_state(
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> OracleRouterStateResponse:
|
||||
"""Decision-maker state, active bindings and recent verified feedback for every ORACLE router. Admin only."""
|
||||
if not _is_admin(user_api_key_dict):
|
||||
raise HTTPException(status_code=403, detail={"error": CommonProxyErrors.not_allowed_access.value})
|
||||
routers: Final = _oracle_routers()
|
||||
if not routers:
|
||||
raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."})
|
||||
return OracleRouterStateResponse(routers=[await candidate.get_state_snapshot() for candidate in routers])
|
||||
|
|
@ -807,6 +807,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( # n
|
|||
from litellm.proxy.management_endpoints.model_management_endpoints import (
|
||||
router as model_management_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.oracle_router_endpoints import (
|
||||
router as oracle_router_management_router,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.organization_endpoints import (
|
||||
router as organization_router,
|
||||
)
|
||||
|
|
@ -22430,6 +22433,7 @@ app.include_router(budget_management_router)
|
|||
app.include_router(model_management_router)
|
||||
app.include_router(model_access_group_management_router)
|
||||
app.include_router(auto_router_management_router)
|
||||
app.include_router(oracle_router_management_router)
|
||||
app.include_router(tag_management_router)
|
||||
app.include_router(workflow_management_router)
|
||||
app.include_router(memory_router)
|
||||
|
|
|
|||
|
|
@ -64,6 +64,7 @@ if TYPE_CHECKING:
|
|||
from litellm.router_strategy.auto_router.auto_router import AutoRouter
|
||||
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
|
||||
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
|
||||
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
|
||||
from litellm.router_strategy.quality_router.quality_router import QualityRouter
|
||||
from litellm.router_utils.cooldown_cache import CooldownCache
|
||||
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets as _AttemptedFallbackTargets
|
||||
|
|
@ -106,6 +107,7 @@ if TYPE_CHECKING:
|
|||
| dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]]
|
||||
| dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]]
|
||||
| dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]
|
||||
| dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]]
|
||||
| list[RoutingPlugin]
|
||||
| RouterBudgetLimiting
|
||||
| DualCache
|
||||
|
|
@ -168,6 +170,7 @@ PluginKind: TypeAlias = Literal[
|
|||
"complexity",
|
||||
"quality",
|
||||
"adaptive",
|
||||
"oracle",
|
||||
"routing",
|
||||
"budget_limiter",
|
||||
"prompt_caching_pins",
|
||||
|
|
@ -1074,6 +1077,11 @@ class RouterRegistry:
|
|||
self, kind: Literal["adaptive"]
|
||||
) -> "dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]": ... # mutable-ok: the live dict
|
||||
|
||||
@overload
|
||||
def plugins(
|
||||
self, kind: Literal["oracle"]
|
||||
) -> "dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]]": ... # mutable-ok: the live dict
|
||||
|
||||
@overload
|
||||
def plugins(self, kind: Literal["routing"]) -> "list[RoutingPlugin]": ... # mutable-ok: the live list
|
||||
|
||||
|
|
@ -1085,7 +1093,7 @@ class RouterRegistry:
|
|||
|
||||
def plugins(self, kind: PluginKind) -> "RegistryPlugins":
|
||||
"""
|
||||
Plugins configured by the deployments: ``auto``, ``complexity``, ``quality`` and ``adaptive``
|
||||
Plugins configured by the deployments: ``auto``, ``complexity``, ``quality``, ``adaptive`` and ``oracle``
|
||||
sub-routers by deployment name, ``routing`` plugins, the deployment ``budget_limiter`` and
|
||||
the ``prompt_caching_pins`` store (were ``auto_routers``, ``complexity_routers``,
|
||||
``quality_routers``, ``adaptive_routers``, ``routing_plugins``,
|
||||
|
|
@ -1100,6 +1108,8 @@ class RouterRegistry:
|
|||
return r._core_quality_routers
|
||||
if kind == "adaptive":
|
||||
return r._core_adaptive_routers
|
||||
if kind == "oracle":
|
||||
return r._core_oracle_routers
|
||||
if kind == "routing":
|
||||
return r._core_routing_plugins
|
||||
if kind == "budget_limiter":
|
||||
|
|
|
|||
|
|
@ -455,6 +455,7 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.router import (
|
||||
CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
|
||||
VALID_LITELLM_ENVIRONMENTS,
|
||||
AdaptiveRouterPreferences,
|
||||
AlertingConfig,
|
||||
AllowedFailsPolicy,
|
||||
AssistantsTypedDict,
|
||||
|
|
@ -559,6 +560,7 @@ if TYPE_CHECKING:
|
|||
from litellm.router_strategy.complexity_router.complexity_router import (
|
||||
ComplexityRouter,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
|
||||
from litellm.router_strategy.quality_router.quality_router import (
|
||||
QualityRouter,
|
||||
)
|
||||
|
|
@ -580,6 +582,7 @@ else:
|
|||
ComplexityRouter = Any
|
||||
AdaptiveRouter = Any
|
||||
QualityRouter = Any
|
||||
OracleRouter = Any # rebind-ok: runtime stand-in for the TYPE_CHECKING-only import above
|
||||
PreRoutingHookResponse = Any
|
||||
|
||||
RouterStrategySelector: TypeAlias = (
|
||||
|
|
@ -1820,6 +1823,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] = {}
|
||||
self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] = {}
|
||||
self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] = {}
|
||||
self._core_oracle_routers: dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]] = {} # mutable-ok: registry
|
||||
self._core_routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else []
|
||||
|
||||
# Initialize model_group_alias early since it's used in set_model_list
|
||||
|
|
@ -2121,6 +2125,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
_routers_holding_global_callbacks.discard(self)
|
||||
slack_alerting_loggers: Final = self._slack_alerting_loggers()
|
||||
adaptive_router_hooks: Final = self._adaptive_router_hooks()
|
||||
oracle_router_hooks: Final = self._oracle_router_hooks()
|
||||
vacated_keys: Final = self._host.callbacks.drop_callbacks(
|
||||
(
|
||||
*self._callback_selectors(),
|
||||
|
|
@ -2128,6 +2133,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
*(self.optional_callbacks or ()),
|
||||
*slack_alerting_loggers,
|
||||
*adaptive_router_hooks,
|
||||
*oracle_router_hooks,
|
||||
),
|
||||
method_owners=(self, *slack_alerting_loggers),
|
||||
)
|
||||
|
|
@ -2140,9 +2146,17 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
adaptive_survivor: Final = next((router for router in survivors if router._core_adaptive_routers), None)
|
||||
if adaptive_survivor is not None and vacated_keys & adaptive_keys:
|
||||
adaptive_survivor._sync_adaptive_router_hooks()
|
||||
oracle_keys: Final = frozenset(map(self._host.callbacks.get_custom_logger_key, oracle_router_hooks))
|
||||
oracle_survivor: Final = next((router for router in survivors if router._core_oracle_routers), None)
|
||||
if oracle_survivor is not None and vacated_keys & oracle_keys:
|
||||
oracle_survivor._sync_oracle_router_hooks()
|
||||
|
||||
def _core_serve_through_request_logging(self) -> None:
|
||||
self._request_scoped_selectors = (*self._callback_selectors(), *self._request_adaptive_router_hooks())
|
||||
self._request_scoped_selectors = (
|
||||
*self._callback_selectors(),
|
||||
*self._request_adaptive_router_hooks(),
|
||||
*self._request_oracle_router_hooks(),
|
||||
)
|
||||
self._request_scoped_policies = (*self._optional_loggers(), *self._slack_alerting_loggers())
|
||||
self._release_global_callbacks()
|
||||
|
||||
|
|
@ -2180,6 +2194,27 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
for tagged in chain.from_iterable(self._core_adaptive_routers.values())
|
||||
)
|
||||
|
||||
def _request_oracle_router_hooks(self) -> tuple[CustomLogger, ...]:
|
||||
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
|
||||
|
||||
return tuple(
|
||||
OracleRouterPostCallHook(oracle_router=tagged.strategy)
|
||||
for tagged in chain.from_iterable(self._core_oracle_routers.values())
|
||||
)
|
||||
|
||||
def _oracle_router_hooks(self) -> tuple[CustomLogger, ...]:
|
||||
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
|
||||
|
||||
own_strategy_ids: Final = frozenset(
|
||||
id(tagged.strategy) for tagged in chain.from_iterable(self._core_oracle_routers.values())
|
||||
)
|
||||
hooks: Final = self._host.callbacks.get_custom_loggers_for_type(OracleRouterPostCallHook)
|
||||
return tuple(
|
||||
hook
|
||||
for hook in hooks
|
||||
if isinstance(hook, OracleRouterPostCallHook) and id(hook.oracle_router) in own_strategy_ids
|
||||
)
|
||||
|
||||
def _routing_callbacks(self) -> Iterable[object]:
|
||||
if not (self._request_scoped_selectors or self._request_scoped_policies):
|
||||
return self._host.callback_list("callbacks")
|
||||
|
|
@ -11606,6 +11641,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._unregister_pre_routing_strategy(registry, model_name, tags)
|
||||
if self._unregister_pre_routing_strategy(self._core_adaptive_routers, model_name, tags):
|
||||
self._sync_adaptive_router_hooks()
|
||||
if self._unregister_pre_routing_strategy(self._core_oracle_routers, model_name, tags):
|
||||
self._sync_oracle_router_hooks()
|
||||
|
||||
def _finalize_adaptive_router_if_configured(self) -> None:
|
||||
"""Locate every adaptive-router deployment in the finalized model_list and
|
||||
|
|
@ -11738,6 +11775,107 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
len(config.available_models),
|
||||
)
|
||||
|
||||
def _is_oracle_router_deployment(self, litellm_params: LiteLLM_Params) -> bool:
|
||||
return classify_strategy_router_model(litellm_params.model) == "oracle"
|
||||
|
||||
def _finalize_oracle_router_if_configured(self) -> None:
|
||||
"""Deferred like the adaptive router, since a `pre_routing` decision maker wraps a strategy router that
|
||||
may appear later in the list. Idempotent across hot reloads, so a router that already holds program
|
||||
bindings and learned state is never rebuilt."""
|
||||
for model_id in self._core_get_model_ids():
|
||||
deployment = self._core_get_deployment(model_id=model_id)
|
||||
if deployment is None or not self._is_oracle_router_deployment(litellm_params=deployment.litellm_params):
|
||||
continue
|
||||
if self._has_registered_strategy(
|
||||
self._core_oracle_routers, deployment.model_name, self._deployment_tags(deployment)
|
||||
):
|
||||
continue
|
||||
self.init_oracle_router_deployment(deployment=deployment)
|
||||
self._sync_oracle_router_hooks()
|
||||
|
||||
def _sync_oracle_router_hooks(self) -> None:
|
||||
"""Exactly one post-call hook per registered oracle router."""
|
||||
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
|
||||
|
||||
for callback in self._host.callbacks.get_custom_loggers_for_type(OracleRouterPostCallHook):
|
||||
self._host.callbacks.remove_callback_from_all_lists(callback)
|
||||
for tagged in chain.from_iterable(self._core_oracle_routers.values()):
|
||||
self._host.callbacks.add_litellm_callback(OracleRouterPostCallHook(oracle_router=tagged.strategy))
|
||||
|
||||
def _resolve_strategy_router(self, model_name: str) -> "PreRoutingStrategy | None":
|
||||
"""The first strategy router registered under `model_name`, for ORACLE's `pre_routing` decision maker."""
|
||||
for registry in (
|
||||
self._core_complexity_routers,
|
||||
self._core_quality_routers,
|
||||
self._core_adaptive_routers,
|
||||
self._core_auto_routers,
|
||||
):
|
||||
tagged_routers = registry.get(model_name)
|
||||
if tagged_routers:
|
||||
return tagged_routers[0].strategy
|
||||
return None
|
||||
|
||||
def _deployment_input_costs(self, model_names: Sequence[str]) -> Mapping[str, float]:
|
||||
"""``input_cost_per_token`` of the first deployment of each model name; litellm_params wins over model_info."""
|
||||
named: Final = (
|
||||
(name, self._core_get_deployment_by_model_group_name(model_group_name=name)) for name in model_names
|
||||
)
|
||||
return MappingProxyType(
|
||||
{
|
||||
name: deployment.model_info.input_cost_per_token
|
||||
for name, deployment in named
|
||||
if deployment is not None and deployment.model_info.input_cost_per_token is not None
|
||||
}
|
||||
)
|
||||
|
||||
def _declared_adaptive_prefs(self, model_name: str) -> AdaptiveRouterPreferences | None:
|
||||
deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name)
|
||||
extra: Final = deployment.model_info.model_extra if deployment is not None else None
|
||||
raw: Final = extra.get("adaptive_router_preferences") if extra else None
|
||||
return None if raw is None else AdaptiveRouterPreferences.model_validate(raw)
|
||||
|
||||
def _deployment_adaptive_prefs(self, model_names: Sequence[str]) -> Mapping[str, AdaptiveRouterPreferences]:
|
||||
"""``model_info.adaptive_router_preferences`` of the first deployment of each model name, where declared."""
|
||||
declared: Final = ((name, self._declared_adaptive_prefs(name)) for name in model_names)
|
||||
return MappingProxyType({name: prefs for name, prefs in declared if prefs is not None})
|
||||
|
||||
def init_oracle_router_deployment(self, deployment: Deployment) -> None:
|
||||
"""Build an OracleRouter from `oracle_router_config` and register it under `deployment.model_name`."""
|
||||
from litellm.router_strategy.oracle_router.decision import build_decision_maker
|
||||
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
|
||||
from litellm.router_strategy.oracle_router.verifier import build_verifier
|
||||
from litellm.types.router import OracleRouterConfig
|
||||
|
||||
raw_config: Final = deployment.litellm_params.oracle_router_config
|
||||
if raw_config is None:
|
||||
raise ValueError("oracle_router_config is required for oracle-router deployments.")
|
||||
config: Final = OracleRouterConfig.model_validate(raw_config)
|
||||
if deployment.model_info.db_model and "custom" in (config.decision_maker.type, config.verifier.type):
|
||||
# A custom decision maker or verifier imports and runs Python named by the deployment. Only the
|
||||
# operator's config file may name it; a model written through the API (any role) must not.
|
||||
raise ValueError(
|
||||
f"oracle_router deployment {deployment.model_name!r}: custom decision makers and verifiers "
|
||||
"can only be configured in the proxy config file, not through the model API."
|
||||
)
|
||||
oracle_router: Final = OracleRouter(
|
||||
router_name=deployment.model_name,
|
||||
config=config,
|
||||
decision_maker=build_decision_maker(
|
||||
config.decision_maker,
|
||||
tuple(config.available_models),
|
||||
self._deployment_input_costs(config.available_models),
|
||||
self._resolve_strategy_router,
|
||||
self._deployment_adaptive_prefs(config.available_models),
|
||||
),
|
||||
verifier=build_verifier(config.verifier),
|
||||
)
|
||||
self._register_pre_routing_strategy(
|
||||
registry=self._core_oracle_routers,
|
||||
deployment=deployment,
|
||||
strategy=oracle_router,
|
||||
strategy_label="Oracle-router",
|
||||
)
|
||||
|
||||
def _is_quality_router_deployment(self, litellm_params: LiteLLM_Params) -> bool:
|
||||
"""
|
||||
Check if the deployment is a quality-router deployment.
|
||||
|
|
@ -11902,6 +12040,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
# Deferred: build the AdaptiveRouter strategy now that all underlying
|
||||
# deployments have been registered.
|
||||
self._finalize_adaptive_router_if_configured()
|
||||
self._finalize_oracle_router_if_configured()
|
||||
|
||||
def _add_deployment(self, deployment: Deployment) -> Deployment:
|
||||
import os
|
||||
|
|
@ -12291,6 +12430,8 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
)
|
||||
):
|
||||
self._finalize_adaptive_router_if_configured()
|
||||
if self._is_oracle_router_deployment(litellm_params=deployment.litellm_params):
|
||||
self._finalize_oracle_router_if_configured()
|
||||
return deployment
|
||||
except Exception as e:
|
||||
if self.ignore_invalid_deployments:
|
||||
|
|
@ -16753,6 +16894,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._core_complexity_routers,
|
||||
self._core_adaptive_routers,
|
||||
self._core_quality_routers,
|
||||
self._core_oracle_routers,
|
||||
)
|
||||
if not any(registries):
|
||||
return None
|
||||
|
|
@ -16854,6 +16996,7 @@ class PythonCore(DeprecatedRouterMembers):
|
|||
self._core_complexity_routers,
|
||||
self._core_adaptive_routers,
|
||||
self._core_quality_routers,
|
||||
self._core_oracle_routers,
|
||||
)
|
||||
):
|
||||
return registered_model_name
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ from litellm.router_strategy.complexity_router.config import (
|
|||
|
||||
AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
|
||||
|
||||
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
|
||||
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality", "oracle"]
|
||||
|
||||
StrategyRouterDependencyRole: TypeAlias = Literal[
|
||||
"tier", "default", "classifier", "embedding", "evaluation", "compactor"
|
||||
|
|
@ -53,6 +53,7 @@ STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset(
|
|||
"adaptive_router_config",
|
||||
"quality_router_config",
|
||||
"quality_router_default_model",
|
||||
"oracle_router_config",
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -65,6 +66,7 @@ _REQUIRED_FIELD_GROUPS: Final[Mapping[StrategyRouterKind, tuple[tuple[str, ...],
|
|||
"complexity": (("complexity_router_config", "complexity_router_default_model"),),
|
||||
"adaptive": (("adaptive_router_config",),),
|
||||
"quality": (("quality_router_config", "quality_router_default_model"),),
|
||||
"oracle": (("oracle_router_config",),),
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -84,6 +86,8 @@ def classify_strategy_router_model(model: str) -> StrategyRouterKind | None:
|
|||
return "adaptive"
|
||||
if remainder.startswith("quality_router"):
|
||||
return "quality"
|
||||
if remainder.startswith("oracle_router"):
|
||||
return "oracle"
|
||||
return "semantic"
|
||||
|
||||
|
||||
|
|
@ -143,6 +147,8 @@ def strategy_router_dependencies(
|
|||
)
|
||||
if kind == "adaptive":
|
||||
return _pool(_mapping(litellm_params.get("adaptive_router_config")).get("available_models"), "tier")
|
||||
if kind == "oracle":
|
||||
return _pool(_mapping(litellm_params.get("oracle_router_config")).get("available_models"), "tier")
|
||||
if kind == "quality":
|
||||
quality: Final = _mapping(litellm_params.get("quality_router_config"))
|
||||
return tuple(
|
||||
|
|
|
|||
101
litellm/router_strategy/oracle_router/README.md
Normal file
101
litellm/router_strategy/oracle_router/README.md
Normal file
|
|
@ -0,0 +1,101 @@
|
|||
# ORACLE router
|
||||
|
||||
Program-level routing for agentic workloads: an agentic task (a *program*) is bound to one model at its
|
||||
first request, every later request with the same `program_id` reuses that binding, and the model is
|
||||
chosen by a pluggable decision maker that learns from a pluggable verifier's delayed verdict.
|
||||
Paper: https://arxiv.org/abs/2607.22465
|
||||
|
||||
## Config
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: oracle-router
|
||||
litellm_params:
|
||||
model: auto_router/oracle_router
|
||||
oracle_router_config:
|
||||
available_models: ["smart", "fast"]
|
||||
decision_maker: { type: thompson, weights: { quality: 0.7, cost: 0.3 } }
|
||||
verifier: { type: guardrail, guardrail: answer-judge }
|
||||
```
|
||||
|
||||
Clients send `metadata.program_id` (or `litellm_session_id`, or the `x-litellm-program-id` header)
|
||||
with every request of a task. Requests without any id are routed statelessly and never learn. Concurrent
|
||||
first requests of one program wait for the decision in flight, so a program is bound exactly once.
|
||||
|
||||
## Decision makers
|
||||
|
||||
| `type` | binds a program by | learns |
|
||||
|---|---|---|
|
||||
| `thompson` (default) | LiteLLM's adaptive-router bandit: a Beta posterior per (request type, model), Thompson sampling, cost-weighted pick with `weights: {quality, cost}` | from verified program scores |
|
||||
| `pre_routing` | the strategy router named by `router` (complexity, quality, adaptive or semantic), once per program | no |
|
||||
| `fixed` | always `model` | no |
|
||||
| `custom` | `path: pkg.mod:Factory`, called with the model tuple; must expose `models`, `async select`, `update`, `snapshot` | up to you |
|
||||
|
||||
## Verifiers
|
||||
|
||||
| `type` | scores a finished program by |
|
||||
|---|---|
|
||||
| `guardrail` | applying the LiteLLM guardrail named `guardrail` to the program's final turn, with the transcript as `structured_messages` and the feedback `payload` as `request_data["metadata"]`: pass = 1, blocked = 0 (read from the status the guardrail records, so an `llm_as_a_judge` with `on_failure: log` still scores 0 when the answer fails). A guardrail that could not evaluate the program, for example a judge whose model was unreachable, counts as a failed verification and teaches the decision maker nothing |
|
||||
| `reported` (default) | `metadata.program_score` on the last request, or `POST /oracle_router/feedback` |
|
||||
| `custom` | `path: pkg.mod:Factory`, called with no arguments; must expose `async verify(outcome)` |
|
||||
|
||||
A verified score `s` adds `s` successes and `1 - s` failures to the cell of the model that served the
|
||||
program. Updates are applied in verification-arrival order. A verifier that raises, or returns a score
|
||||
that is not a finite number, counts as a failed verification (`verifications_failed` in the state) and
|
||||
never reaches the decision maker.
|
||||
|
||||
## Finishing a program
|
||||
|
||||
Either mark the last request with `metadata.program_done: true` (plus `program_score` for the reported
|
||||
verifier), or call `POST /oracle_router/feedback {"program_id": ..., "score": ..., "cost": ...}` from
|
||||
the harness once the task is graded. Feedback that arrives before any response of the program was
|
||||
observed (it raced the last response's logging, or every request failed) is deferred: the reply says
|
||||
`deferred: true` and the next observed response completes the program with it, so nothing is ever
|
||||
verified or learned without an answer. A guardrail verifier also fails the verification of a program
|
||||
whose final response has no text, rather than reading a judge's skip as a pass. `GET /oracle_router/state`
|
||||
(admin) shows bindings, decision-maker state and recent feedback. The response header
|
||||
`x-litellm-oracle-router-model` names the bound model.
|
||||
|
||||
Verification runs in the background and is bounded: at most `max_pending_verifications` (default 64)
|
||||
verifications are in flight per router, and each one is given `verification_timeout_seconds` (default
|
||||
120) before it counts as failed. While the backlog is full a completion is refused and the program stays
|
||||
bound, so it can be completed again once the backlog drains: the feedback endpoint answers 503 and a
|
||||
`program_done` request records the refusal in `last_failure`; `verifications_rejected` in the state
|
||||
counts them.
|
||||
|
||||
## Security
|
||||
|
||||
A program is private to the API key that started it: the same `program_id` under another key is another
|
||||
program, so no caller can ride on, observe or complete someone else's binding. `POST /oracle_router/feedback`
|
||||
completes the caller's own program; an admin key may complete any key's. The post-call hook only reads
|
||||
requests the Router stamped as routed by this router (`routing_decision`), so ORACLE keys placed in the
|
||||
metadata of a request to some other model are ignored, and the router drops any caller-supplied copy of
|
||||
its internal metadata keys before stamping its own.
|
||||
|
||||
A `pre_routing` decision maker hands the strategy router it wraps what LiteLLM forwards to any detached
|
||||
sub-call: the caller's identity (`user_api_key`, its hash and alias, team, user, organization and end user,
|
||||
and the auth object the Router filters deployments by, minus the request's budget reservation), nothing else
|
||||
of the request, and the program's `turn_off_message_logging` opt-out. A classifier call made on the program's
|
||||
behalf is therefore billed to the caller's key and team and never logs a prompt the caller marked private;
|
||||
the identity is used for that call only and is not kept on the program. Verification runs under the
|
||||
program's opt-out as well: a program is private once any of its requests opted out, whether or not that
|
||||
request produced an answer, so a `guardrail` verifier's judge call logs neither the prompt nor the
|
||||
transcript. LiteLLM redacts a private request's messages and answer before the router's hook sees them, so
|
||||
a `guardrail` verifier cannot judge a program whose final turn was redacted: that is a failed verification
|
||||
and teaches the decision maker nothing. Use the `reported` verifier for routers whose programs opt out of
|
||||
logging. A `guardrail` verifier's judge call carries no caller identity, as the proxy's own
|
||||
`llm_as_a_judge` flow does not, so its spend is not attributed to the program's key or team. The feedback
|
||||
`payload` reaches the guardrail with the proxy-owned keys removed (`user_api_key_*`, the sub-call origin,
|
||||
headers and the routing decision), so a caller cannot pose as the proxy to a guardrail that forwards
|
||||
identity to its vendor.
|
||||
|
||||
`custom` decision makers and verifiers import Python named by the deployment, so they are honoured only
|
||||
for deployments in the proxy config file. A model written through the API (`/model/new`, `/model/update`,
|
||||
whatever the caller's role) with a `custom` type is rejected when the router is built and never imported.
|
||||
|
||||
## Limits
|
||||
|
||||
State is in memory per proxy process; it is not persisted or shared across replicas. The paper's
|
||||
dispatch scheduler (DISC) is not part of this strategy. The router and its hook write nothing to the
|
||||
process log: `GET /oracle_router/state` carries the counters and the last background failure
|
||||
(`last_failure`) instead.
|
||||
32
litellm/router_strategy/oracle_router/__init__.py
Normal file
32
litellm/router_strategy/oracle_router/__init__.py
Normal file
|
|
@ -0,0 +1,32 @@
|
|||
from litellm.router_strategy.oracle_router.decision import (
|
||||
DecisionMaker,
|
||||
FixedDecisionMaker,
|
||||
PreRoutingDecisionMaker,
|
||||
ProgramContext,
|
||||
ThompsonDecisionMaker,
|
||||
build_decision_maker,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, ProgramBinding
|
||||
from litellm.router_strategy.oracle_router.verifier import (
|
||||
GuardrailVerifier,
|
||||
ProgramOutcome,
|
||||
ReportedVerifier,
|
||||
Verifier,
|
||||
build_verifier,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DecisionMaker",
|
||||
"FixedDecisionMaker",
|
||||
"GuardrailVerifier",
|
||||
"OracleRouter",
|
||||
"PreRoutingDecisionMaker",
|
||||
"ProgramBinding",
|
||||
"ProgramContext",
|
||||
"ProgramOutcome",
|
||||
"ReportedVerifier",
|
||||
"ThompsonDecisionMaker",
|
||||
"Verifier",
|
||||
"build_decision_maker",
|
||||
"build_verifier",
|
||||
]
|
||||
18
litellm/router_strategy/oracle_router/config.py
Normal file
18
litellm/router_strategy/oracle_router/config.py
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
"""Metadata keys and headers shared by the ORACLE router, its post-call hook and the proxy endpoints."""
|
||||
|
||||
from typing import Final
|
||||
|
||||
ORACLE_ROUTER_MODEL_PREFIX: Final[str] = "auto_router/oracle_router"
|
||||
|
||||
CHOSEN_MODEL_METADATA_KEY: Final[str] = "oracle_router_chosen_model"
|
||||
PROGRAM_ID_METADATA_KEY: Final[str] = "oracle_router_program_id"
|
||||
RESPONSE_HEADER: Final[str] = "x-litellm-oracle-router-model"
|
||||
PROGRAM_ID_HEADER: Final[str] = "x-litellm-program-id"
|
||||
|
||||
PROGRAM_DONE_KEY: Final[str] = "program_done"
|
||||
PROGRAM_SCORE_KEY: Final[str] = "program_score"
|
||||
|
||||
SESSION_ID_FALLBACK_KEYS: Final[tuple[str, ...]] = ("litellm_session_id", "session_id")
|
||||
|
||||
FEEDBACK_HISTORY_SIZE: Final[int] = 200
|
||||
PROGRAM_SWEEP_THRESHOLD: Final[int] = 1024
|
||||
248
litellm/router_strategy/oracle_router/decision.py
Normal file
248
litellm/router_strategy/oracle_router/decision.py
Normal file
|
|
@ -0,0 +1,248 @@
|
|||
"""Decision makers: the pluggable policy that binds a program to a model.
|
||||
|
||||
``DecisionMaker`` is the whole contract. The default, ``thompson``, is LiteLLM's own adaptive-router
|
||||
bandit (a Beta posterior per request type and model, Thompson sampling, cost-weighted pick) trained on
|
||||
verified program outcomes instead of per-turn regex signals. ``pre_routing`` lets any LiteLLM strategy
|
||||
router (complexity, quality, adaptive, semantic) make the proposal once per program, ``fixed`` pins one
|
||||
model, and ``custom`` imports an operator's class.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import itertools
|
||||
import random
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
|
||||
from litellm.router_strategy.adaptive_router.bandit import BanditCell, apply_delta, initial_cell, pick_best
|
||||
from litellm.types.router import (
|
||||
AdaptiveRouterPreferences,
|
||||
AdaptiveRouterWeights,
|
||||
OracleDecisionMakerConfig,
|
||||
PreRoutingStrategy,
|
||||
RequestType,
|
||||
)
|
||||
|
||||
_NO_CALLER: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProgramContext:
|
||||
"""What a decision maker sees when a program arrives: its initial request and LiteLLM's request type."""
|
||||
|
||||
program_id: str
|
||||
prompt: str
|
||||
request_type: RequestType
|
||||
message_logging_off: bool = False
|
||||
caller: Mapping[str, object] = field(default_factory=lambda: _NO_CALLER)
|
||||
"""The caller's identity metadata, so a sub-call made on the program's behalf is billed to the caller."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class DecisionMaker(Protocol):
|
||||
@property
|
||||
def models(self) -> tuple[str, ...]: ...
|
||||
|
||||
async def select(self, context: ProgramContext) -> str: ...
|
||||
|
||||
def update(self, context: ProgramContext, model: str, score: float) -> None: ...
|
||||
|
||||
def snapshot(self) -> Mapping[str, object]: ...
|
||||
|
||||
|
||||
def _check_model(models: Sequence[str], model: str) -> None:
|
||||
if model not in models:
|
||||
raise KeyError(f"unknown model {model!r}; configured models: {list(models)}")
|
||||
|
||||
|
||||
_UNIFORM_PRIOR: Final = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
|
||||
|
||||
|
||||
class ThompsonDecisionMaker:
|
||||
"""LiteLLM's adaptive-router bandit, keyed by (request type, model) and updated with verified scores.
|
||||
|
||||
Cells start from the same priors the adaptive router uses, read from each model's
|
||||
``model_info.adaptive_router_preferences`` (tier 2 when a model declares none). A verified score ``s``
|
||||
in [0, 1] adds ``s`` successes and ``1 - s`` failures to the cell of the model that served the program.
|
||||
Selection samples every cell once and picks the best quality/cost score with the adaptive router's
|
||||
weights, so ``weights.cost`` is the only cost knob.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
models: Sequence[str],
|
||||
model_costs: Mapping[str, float],
|
||||
weights: AdaptiveRouterWeights | None = None,
|
||||
model_prefs: Mapping[str, AdaptiveRouterPreferences] | None = None,
|
||||
seed: int = 0,
|
||||
) -> None:
|
||||
if not models:
|
||||
raise ValueError("a decision maker needs at least one model")
|
||||
self._models: Final[tuple[str, ...]] = tuple(models)
|
||||
costs: Final = {m: float(model_costs.get(m, 0.0)) for m in self._models}
|
||||
self._costs: Final[dict[str, float]] = costs # mutable-ok: pick_best takes a dict
|
||||
self._weights: Final[AdaptiveRouterWeights] = weights or AdaptiveRouterWeights()
|
||||
self._rng: Final = random.Random(seed)
|
||||
prefs: Final = model_prefs or {}
|
||||
grid: Final = tuple(itertools.product(RequestType, self._models))
|
||||
cells: Final = {(rt, model): initial_cell(prefs.get(model, _UNIFORM_PRIOR), rt) for rt, model in grid}
|
||||
self._cells: Final[dict[tuple[RequestType, str], BanditCell]] = cells # mutable-ok: replaced on update
|
||||
self.pulls: Final[dict[str, int]] = {m: 0 for m in self._models} # mutable-ok: counters
|
||||
|
||||
@property
|
||||
def models(self) -> tuple[str, ...]:
|
||||
return self._models
|
||||
|
||||
async def select(self, context: ProgramContext) -> str:
|
||||
cells: Final = {model: self._cells[(context.request_type, model)] for model in self._models}
|
||||
chosen: Final = pick_best(cells, self._costs, self._weights.quality, self._weights.cost, rng=self._rng)
|
||||
self.pulls[chosen] += 1
|
||||
return chosen
|
||||
|
||||
def update(self, context: ProgramContext, model: str, score: float) -> None:
|
||||
_check_model(self._models, model)
|
||||
key: Final = (context.request_type, model)
|
||||
self._cells[key] = apply_delta(self._cells[key], score, 1.0 - score)
|
||||
|
||||
def estimate(self, context: ProgramContext, model: str) -> float:
|
||||
_check_model(self._models, model)
|
||||
return self._cells[(context.request_type, model)].mean
|
||||
|
||||
def snapshot(self) -> Mapping[str, object]:
|
||||
cells: Final = {
|
||||
f"{request_type.value}/{model}": {"mean": round(cell.mean, 4), "samples": cell.total_samples}
|
||||
for (request_type, model), cell in self._cells.items()
|
||||
if cell.total_samples > 0
|
||||
}
|
||||
return {
|
||||
"type": "thompson",
|
||||
"weights": {"quality": self._weights.quality, "cost": self._weights.cost},
|
||||
"pulls": dict(self.pulls),
|
||||
"cells": cells,
|
||||
}
|
||||
|
||||
|
||||
class FixedDecisionMaker:
|
||||
"""Always the same model; the single-model baseline that still gets verified and reported."""
|
||||
|
||||
def __init__(self, models: Sequence[str], model: str) -> None:
|
||||
_check_model(models, model)
|
||||
self._models: Final[tuple[str, ...]] = tuple(models)
|
||||
self._model: Final[str] = model
|
||||
self._updates: int = 0
|
||||
|
||||
@property
|
||||
def models(self) -> tuple[str, ...]:
|
||||
return self._models
|
||||
|
||||
async def select(self, context: ProgramContext) -> str:
|
||||
return self._model
|
||||
|
||||
def update(self, context: ProgramContext, model: str, score: float) -> None:
|
||||
_check_model(self._models, model)
|
||||
self._updates += 1
|
||||
|
||||
def snapshot(self) -> Mapping[str, object]:
|
||||
return {"type": "fixed", "model": self._model, "updates": self._updates}
|
||||
|
||||
|
||||
def strategy_request_kwargs(context: ProgramContext) -> dict[str, object]: # mutable-ok: pre-routing contract
|
||||
"""The request a wrapped strategy router sees for the program: the caller's identity, so a classifier call
|
||||
the strategy makes is billed to the caller's key and team, and the program's message-logging opt-out where
|
||||
LiteLLM reads it. A fresh dict per call: the strategies write their decision into the bucket they are handed."""
|
||||
privacy: Final[Mapping[str, object]] = {"turn_off_message_logging": True} if context.message_logging_off else {}
|
||||
return {"metadata": {**context.caller, **privacy}}
|
||||
|
||||
|
||||
class PreRoutingDecisionMaker:
|
||||
"""Let one of LiteLLM's own strategy routers propose the model, once per program.
|
||||
|
||||
The wrapped router (complexity, quality, adaptive or semantic) sees the program's initial request as a
|
||||
single user message. Its per-request decision becomes a per-program binding, and the program's verified
|
||||
outcome is recorded against that model, so the state endpoint shows what the strategy actually achieved.
|
||||
"""
|
||||
|
||||
def __init__(self, models: Sequence[str], router_name: str, strategy: PreRoutingStrategy) -> None:
|
||||
if not models:
|
||||
raise ValueError("a decision maker needs at least one model")
|
||||
self._models: Final[tuple[str, ...]] = tuple(models)
|
||||
self._router_name: Final[str] = router_name
|
||||
self._strategy: Final[PreRoutingStrategy] = strategy
|
||||
self._proposals: Final[dict[str, int]] = {} # mutable-ok: proposal counts per model, incremented per program
|
||||
self._fallbacks: int = 0
|
||||
self._updates: int = 0
|
||||
|
||||
@property
|
||||
def models(self) -> tuple[str, ...]:
|
||||
return self._models
|
||||
|
||||
async def select(self, context: ProgramContext) -> str:
|
||||
with inherit_message_logging_privacy(context.message_logging_off):
|
||||
response: Final = await self._strategy.async_pre_routing_hook(
|
||||
model=self._router_name,
|
||||
request_kwargs=strategy_request_kwargs(context),
|
||||
messages=[{"role": "user", "content": context.prompt}],
|
||||
)
|
||||
proposed: Final = response.model if response is not None and response.model in self._models else None
|
||||
if proposed is None:
|
||||
self._fallbacks += 1
|
||||
chosen: Final = proposed if proposed is not None else self._models[0]
|
||||
self._proposals[chosen] = self._proposals.get(chosen, 0) + 1
|
||||
return chosen
|
||||
|
||||
def update(self, context: ProgramContext, model: str, score: float) -> None:
|
||||
_check_model(self._models, model)
|
||||
self._updates += 1
|
||||
|
||||
def snapshot(self) -> Mapping[str, object]:
|
||||
return {
|
||||
"type": "pre_routing",
|
||||
"router": self._router_name,
|
||||
"proposals": dict(self._proposals),
|
||||
"fallbacks": self._fallbacks,
|
||||
"updates": self._updates,
|
||||
}
|
||||
|
||||
|
||||
def load_object(path: str) -> object:
|
||||
"""Import ``package.module:attribute``."""
|
||||
module_name, separator, attribute = path.partition(":")
|
||||
if not separator or not module_name or not attribute:
|
||||
raise ValueError(f"expected 'package.module:Attribute', got {path!r}")
|
||||
loaded: Final[object] = getattr(importlib.import_module(module_name), attribute) # pyright: ignore[reportAny] # module attributes are untyped
|
||||
return loaded
|
||||
|
||||
|
||||
def build_decision_maker(
|
||||
config: OracleDecisionMakerConfig,
|
||||
models: Sequence[str],
|
||||
model_costs: Mapping[str, float],
|
||||
resolve_strategy: Callable[[str], PreRoutingStrategy | None],
|
||||
model_prefs: Mapping[str, AdaptiveRouterPreferences] | None = None,
|
||||
) -> DecisionMaker:
|
||||
"""Instantiate the configured decision maker over ``models`` (strongest first)."""
|
||||
match config.type:
|
||||
case "thompson":
|
||||
return ThompsonDecisionMaker(
|
||||
models, model_costs, weights=config.weights, model_prefs=model_prefs, seed=config.seed
|
||||
)
|
||||
case "fixed":
|
||||
return FixedDecisionMaker(models, model=config.model or models[0])
|
||||
case "pre_routing":
|
||||
router_name: Final = config.router or ""
|
||||
strategy: Final = resolve_strategy(router_name)
|
||||
if strategy is None:
|
||||
raise ValueError(
|
||||
f"oracle_router decision_maker.router {router_name!r} is not a configured strategy router"
|
||||
)
|
||||
return PreRoutingDecisionMaker(models, router_name=router_name, strategy=strategy)
|
||||
case "custom":
|
||||
factory: Final = load_object(config.path or "")
|
||||
if not callable(factory):
|
||||
raise TypeError(f"oracle_router decision_maker.path {config.path!r} is not callable")
|
||||
instance: Final = factory(tuple(models))
|
||||
if not isinstance(instance, DecisionMaker):
|
||||
raise TypeError(f"{config.path!r} must build an object with select, update, snapshot and models")
|
||||
return instance
|
||||
119
litellm/router_strategy/oracle_router/hooks.py
Normal file
119
litellm/router_strategy/oracle_router/hooks.py
Normal file
|
|
@ -0,0 +1,119 @@
|
|||
"""Post-call hook for the ORACLE router.
|
||||
|
||||
After every completion it adds LiteLLM's computed spend and the latest answer to the program's row, and
|
||||
when the request carried ``metadata.program_done`` it completes the program: the slot is released at once
|
||||
and the verifier runs in the background. Only requests the Router stamped as routed by this ORACLE router
|
||||
are read, and only against the program of the key that sent them. Nothing here can fail a request, and
|
||||
nothing here logs: a failure is kept on the router for ``/oracle_router/state``.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.router_strategy.oracle_router.config import (
|
||||
CHOSEN_MODEL_METADATA_KEY,
|
||||
PROGRAM_DONE_KEY,
|
||||
PROGRAM_ID_METADATA_KEY,
|
||||
PROGRAM_SCORE_KEY,
|
||||
RESPONSE_HEADER,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, as_str_mapping, logged_request_is_private
|
||||
from litellm.router_strategy.oracle_router.verifier import response_text
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
|
||||
|
||||
def _request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""The request's metadata as the proxy saw it: the caller's bucket with the proxy-internal one on top."""
|
||||
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params"))
|
||||
return _merged(litellm_params.get("metadata"), litellm_params.get("litellm_metadata"))
|
||||
|
||||
|
||||
def _merged(metadata: object, litellm_metadata: object) -> Mapping[str, object]:
|
||||
return {**as_str_mapping(metadata), **as_str_mapping(litellm_metadata)}
|
||||
|
||||
|
||||
def _routed_by(metadata: Mapping[str, object], router_name: str) -> bool:
|
||||
"""True when the Router stamped this request as routed by this ORACLE router.
|
||||
|
||||
The Router writes or clears ``routing_decision`` on every routing attempt, so a caller cannot make a
|
||||
request to some other model look like one of this router's by putting ORACLE keys in its metadata.
|
||||
"""
|
||||
decision: Final = as_str_mapping(metadata.get("routing_decision"))
|
||||
return decision.get("router_type") == "oracle" and decision.get("router_model_name") == router_name
|
||||
|
||||
|
||||
def _owner(metadata: Mapping[str, object]) -> str | None:
|
||||
"""The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy."""
|
||||
value: Final = metadata.get("user_api_key_hash")
|
||||
return str(value) if value else None
|
||||
|
||||
|
||||
def _score(metadata: Mapping[str, object]) -> float | None:
|
||||
raw: Final = metadata.get(PROGRAM_SCORE_KEY)
|
||||
if isinstance(raw, bool) or not isinstance(raw, (int, float, str)):
|
||||
return None
|
||||
try:
|
||||
score: Final = float(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
return score if math.isfinite(score) else None # a NaN would corrupt a bandit cell for good
|
||||
|
||||
|
||||
class OracleRouterPostCallHook(CustomLogger):
|
||||
"""One hook per OracleRouter, registered in ``litellm.callbacks``."""
|
||||
|
||||
def __init__(self, oracle_router: OracleRouter) -> None:
|
||||
self.oracle_router: Final[OracleRouter] = oracle_router
|
||||
|
||||
async def async_post_call_response_headers_hook(
|
||||
self,
|
||||
data: Mapping[str, object],
|
||||
user_api_key_dict: "UserAPIKeyAuth",
|
||||
response: object,
|
||||
request_headers: dict[str, str] | None = None, # mutable-ok: CustomLogger's hook signature
|
||||
litellm_call_info: dict[str, object] | None = None, # mutable-ok: CustomLogger's hook signature
|
||||
) -> dict[str, str] | None: # mutable-ok: CustomLogger's hook signature
|
||||
metadata: Final = _merged(data.get("metadata"), data.get("litellm_metadata"))
|
||||
if not _routed_by(metadata, self.oracle_router.router_name):
|
||||
return None
|
||||
chosen: Final = metadata.get(CHOSEN_MODEL_METADATA_KEY)
|
||||
return {RESPONSE_HEADER: str(chosen)} if chosen else None
|
||||
|
||||
async def async_log_success_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self._record(kwargs, response_obj, succeeded=True)
|
||||
|
||||
async def async_log_failure_event(
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
|
||||
) -> None:
|
||||
self._record(kwargs, response_obj, succeeded=False)
|
||||
|
||||
def _record(self, kwargs: Mapping[str, object], response_obj: object, succeeded: bool) -> None:
|
||||
try:
|
||||
metadata: Final = _request_metadata(kwargs)
|
||||
if not _routed_by(metadata, self.oracle_router.router_name):
|
||||
return
|
||||
program_id: Final = metadata.get(PROGRAM_ID_METADATA_KEY)
|
||||
if not isinstance(program_id, str):
|
||||
return
|
||||
owner: Final = _owner(metadata)
|
||||
if logged_request_is_private(kwargs):
|
||||
self.oracle_router.mark_message_logging_off(program_id, owner)
|
||||
if succeeded:
|
||||
cost: Final = kwargs.get("response_cost")
|
||||
self.oracle_router.observe_request(
|
||||
program_id,
|
||||
cost=float(cost) if isinstance(cost, (int, float)) else 0.0,
|
||||
response_text=response_text(response_obj),
|
||||
messages=kwargs.get("messages"),
|
||||
owner=owner,
|
||||
)
|
||||
if metadata.get(PROGRAM_DONE_KEY):
|
||||
self.oracle_router.complete(program_id, score=_score(metadata), owner=owner)
|
||||
except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes
|
||||
self.oracle_router.note_failure("post-call hook", error)
|
||||
548
litellm/router_strategy/oracle_router/oracle_router.py
Normal file
548
litellm/router_strategy/oracle_router/oracle_router.py
Normal file
|
|
@ -0,0 +1,548 @@
|
|||
"""The ORACLE router: one binding per agentic program, learned from verified, delayed feedback.
|
||||
|
||||
A program is one agentic task made of many LLM requests that share a prefix. Its first request is bound
|
||||
to a model by the decision maker and every later request with the same program id reuses that binding.
|
||||
When the program finishes, the verifier scores it off the critical path, the score and LiteLLM's spend
|
||||
for the program become one reward, and the decision maker learns from it in arrival order.
|
||||
|
||||
Paper: ORACLE, https://arxiv.org/abs/2607.22465
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from types import MappingProxyType
|
||||
from typing import Final, NamedTuple
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict
|
||||
)
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
inherit_message_logging_privacy,
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
|
||||
from litellm.litellm_core_utils.redact_messages import (
|
||||
should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # upstream helper takes a bare dict
|
||||
)
|
||||
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
|
||||
from litellm.router_strategy.oracle_router.config import (
|
||||
CHOSEN_MODEL_METADATA_KEY,
|
||||
FEEDBACK_HISTORY_SIZE,
|
||||
PROGRAM_ID_HEADER,
|
||||
PROGRAM_ID_METADATA_KEY,
|
||||
PROGRAM_SWEEP_THRESHOLD,
|
||||
SESSION_ID_FALLBACK_KEYS,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.decision import DecisionMaker, ProgramContext
|
||||
from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, Verifier, clamp_score
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import OracleRouterConfig, PreRoutingHookResponse, RequestType
|
||||
from litellm.types.utils import (
|
||||
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
|
||||
RoutingDecisionCause,
|
||||
StandardLoggingRoutingDecision,
|
||||
)
|
||||
|
||||
_LEARNING_DECISION_MAKERS: Final[frozenset[str]] = frozenset({"thompson"})
|
||||
_MESSAGES: Final = TypeAdapter(Sequence[AllMessageValues])
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_OBJECTS: Final = TypeAdapter(Sequence[object])
|
||||
|
||||
|
||||
def as_str_mapping(value: object) -> Mapping[str, object]:
|
||||
"""A typed, read-only view of a request-shaped mapping; empty for anything else."""
|
||||
if not isinstance(value, Mapping):
|
||||
return _EMPTY
|
||||
try:
|
||||
return _STR_MAPPING.validate_python(value)
|
||||
except ValidationError:
|
||||
return _EMPTY
|
||||
|
||||
|
||||
class ProgramKey(NamedTuple):
|
||||
"""A program is private to the API key that started it: the same id under another key is another program."""
|
||||
|
||||
owner: str | None
|
||||
program_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DeferredCompletion:
|
||||
"""A completion that arrived before any response of the program was observed; the next one applies it."""
|
||||
|
||||
score: float | None
|
||||
cost: float | None
|
||||
payload: Mapping[str, object]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProgramBinding:
|
||||
"""One row of the lookup table: program id to model, plus what the proxy has seen of the program."""
|
||||
|
||||
program_id: str
|
||||
model: str
|
||||
context: ProgramContext
|
||||
api_key_hash: str | None
|
||||
bound_at: float
|
||||
requests: int = 0
|
||||
cost: float = 0.0
|
||||
last_response_text: str = ""
|
||||
last_messages: Sequence[AllMessageValues] = ()
|
||||
deferred: DeferredCompletion | None = None
|
||||
message_logging_off: bool = False
|
||||
"""True once any request of the program opted out of message logging; verification then runs privately."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FeedbackRecord:
|
||||
program_id: str
|
||||
model: str
|
||||
request_type: str
|
||||
score: float
|
||||
cost: float
|
||||
verified_at: float
|
||||
|
||||
|
||||
def program_messages(
|
||||
messages: list[dict[str, object]] | None, # mutable-ok: pre-routing contract
|
||||
input: str | list[object] | None, # mutable-ok: pre-routing contract
|
||||
request_kwargs: Mapping[str, object],
|
||||
) -> Sequence[Mapping[str, object]] | None:
|
||||
"""The request's messages in chat shape whichever surface sent them: a Responses API ``input`` is converted
|
||||
the way guardrails convert it, so a program started through ``/v1/responses`` is bound on its real prompt."""
|
||||
if messages or input is None:
|
||||
return messages
|
||||
return resolve_structured_messages(
|
||||
None,
|
||||
{**request_kwargs, "input": input, "litellm_metadata": as_str_mapping(request_kwargs.get("litellm_metadata"))},
|
||||
)
|
||||
|
||||
|
||||
def first_user_text(messages: Sequence[Mapping[str, object]] | None) -> str:
|
||||
"""The program's initial request: the first user message, text parts only."""
|
||||
for message in messages or ():
|
||||
if message.get("role") != "user":
|
||||
continue
|
||||
content = message.get("content")
|
||||
if isinstance(content, str):
|
||||
return content
|
||||
if isinstance(content, Sequence):
|
||||
parts = tuple(as_str_mapping(part) for part in _OBJECTS.validate_python(content))
|
||||
return "\n".join(str(part.get("text", "")) for part in parts if part.get("type") == "text")
|
||||
return ""
|
||||
return ""
|
||||
|
||||
|
||||
def _metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""The request's metadata as callers see it, with the proxy-internal bucket layered on top."""
|
||||
return {**as_str_mapping(request_kwargs.get("metadata")), **as_str_mapping(request_kwargs.get("litellm_metadata"))}
|
||||
|
||||
|
||||
def _header(request_kwargs: Mapping[str, object], name: str) -> str | None:
|
||||
headers: Final = as_str_mapping(request_kwargs.get("headers"))
|
||||
matches: Final = tuple(value for key, value in headers.items() if key.lower() == name)
|
||||
return str(matches[0]) if matches else None
|
||||
|
||||
|
||||
def resolve_program_id(request_kwargs: Mapping[str, object], program_id_key: str) -> str | None:
|
||||
"""The program id of a request: ``metadata[program_id_key]``, then the session id spellings, then the header."""
|
||||
metadata: Final = _metadata(request_kwargs)
|
||||
candidates: Final = (
|
||||
metadata.get(program_id_key),
|
||||
*(metadata.get(key) for key in SESSION_ID_FALLBACK_KEYS),
|
||||
request_kwargs.get("litellm_session_id"),
|
||||
_header(request_kwargs, PROGRAM_ID_HEADER),
|
||||
)
|
||||
return next((str(candidate) for candidate in candidates if candidate), None)
|
||||
|
||||
|
||||
def _transcript(messages: object, fallback: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]:
|
||||
if not messages:
|
||||
return fallback
|
||||
try:
|
||||
return tuple(_MESSAGES.validate_python(messages))
|
||||
except ValidationError:
|
||||
return fallback
|
||||
|
||||
|
||||
def request_is_private(request_kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether LiteLLM will redact this request's messages when it logs them, by the logger's own rule: the
|
||||
request's ``turn_off_message_logging``, its redaction headers, or the global setting."""
|
||||
kwargs: Final = dict(request_kwargs)
|
||||
return _redacts(
|
||||
{
|
||||
"litellm_params": kwargs,
|
||||
"standard_callback_dynamic_params": initialize_standard_callback_dynamic_params(kwargs),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def logged_request_is_private(kwargs: Mapping[str, object]) -> bool:
|
||||
"""Whether LiteLLM redacted a logged request's messages, from the kwargs it hands a logging callback."""
|
||||
return _redacts(
|
||||
{
|
||||
**kwargs,
|
||||
"standard_callback_dynamic_params": kwargs.get("standard_callback_dynamic_params")
|
||||
or initialize_standard_callback_dynamic_params(dict(kwargs)),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _redacts(model_call_details: dict[str, object]) -> bool: # mutable-ok: the logger's own parameter shape
|
||||
try:
|
||||
return should_redact_message_logging(model_call_details)
|
||||
except AttributeError: # malformed metadata makes the logger raise and log nothing, so the program stays private
|
||||
return True
|
||||
|
||||
|
||||
def forwardable_caller(request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
|
||||
"""What a classifier call made on the program's behalf inherits from the request that bound it: the caller's
|
||||
identity from whichever metadata bucket the surface used, with its budget reservation removed, stamped as the
|
||||
sub-call it becomes. The routing decision, the logging payload and every other per-request state stay behind."""
|
||||
return MappingProxyType(
|
||||
sanitized_forwardable_call_metadata(_metadata(request_kwargs), AUTOROUTER_CLASSIFIER_CALL_ORIGIN)
|
||||
)
|
||||
|
||||
|
||||
def _api_key_hash(request_kwargs: Mapping[str, object]) -> str | None:
|
||||
"""The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy."""
|
||||
value: Final = _metadata(request_kwargs).get("user_api_key_hash")
|
||||
return str(value) if value else None
|
||||
|
||||
|
||||
def _stamp_internal(
|
||||
request_kwargs: dict[str, object], # mutable-ok: pre-routing contract
|
||||
key: str,
|
||||
value: object | None,
|
||||
) -> None:
|
||||
"""Write a router-internal metadata key where the proxy keeps its own state.
|
||||
|
||||
Any copy the caller supplied is dropped from both buckets first, so the post-call hook only ever reads
|
||||
what this router wrote; None clears the key.
|
||||
"""
|
||||
for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
|
||||
if isinstance(bucket, dict):
|
||||
bucket.pop(key, None) # pyright: ignore[reportUnknownMemberType] # caller-supplied dict
|
||||
if value is not None:
|
||||
_, internal = get_or_create_metadata_bucket(request_kwargs) # pyright: ignore[reportUnknownVariableType] # bare dict upstream
|
||||
internal[key] = value
|
||||
|
||||
|
||||
class VerificationBacklogFull(RuntimeError):
|
||||
"""``max_pending_verifications`` verifications are in flight; the program stays bound for a later retry."""
|
||||
|
||||
|
||||
class OracleRouter:
|
||||
"""One instance per ``auto_router/oracle_router`` deployment."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
router_name: str,
|
||||
config: OracleRouterConfig,
|
||||
decision_maker: DecisionMaker,
|
||||
verifier: Verifier,
|
||||
clock: Callable[[], float] = time.time,
|
||||
) -> None:
|
||||
self.router_name: Final[str] = router_name
|
||||
self.config: Final[OracleRouterConfig] = config
|
||||
self.decision_maker: Final[DecisionMaker] = decision_maker
|
||||
self.verifier: Final[Verifier] = verifier
|
||||
self._clock: Final = clock
|
||||
self._cause: Final[RoutingDecisionCause] = (
|
||||
"bandit" if config.decision_maker.type in _LEARNING_DECISION_MAKERS else "classifier_plugin"
|
||||
)
|
||||
self._bindings: Final[dict[ProgramKey, ProgramBinding]] = {} # mutable-ok: the lookup table
|
||||
self._binding_in_flight: Final[dict[ProgramKey, asyncio.Event]] = {} # mutable-ok: being bound right now
|
||||
self._tasks: Final[set[asyncio.Task[float]]] = set() # mutable-ok: in-flight verifications
|
||||
self._history: Final[deque[FeedbackRecord]] = deque(maxlen=FEEDBACK_HISTORY_SIZE) # mutable-ok: bounded log
|
||||
self._update_lock: Final = asyncio.Lock()
|
||||
self.programs_bound: int = 0
|
||||
self.programs_completed: int = 0
|
||||
self.programs_evicted: int = 0
|
||||
self.stateless_requests: int = 0
|
||||
self.verifications_failed: int = 0
|
||||
self.verifications_rejected: int = 0
|
||||
self.last_failure: str | None = None
|
||||
|
||||
@property
|
||||
def models(self) -> tuple[str, ...]:
|
||||
return tuple(self.config.available_models)
|
||||
|
||||
def note_failure(self, step: str, error: Exception) -> None:
|
||||
"""Keep the last failure of a background step for ``/oracle_router/state``.
|
||||
|
||||
The router and its hook write nothing to the process log: every request input reaches them, and a
|
||||
log call on that path makes CodeQL's log-injection analysis fall back to a whole-codebase pass that
|
||||
no longer fits its limits on this codebase. The admin state endpoint carries the failure instead.
|
||||
"""
|
||||
self.last_failure = f"{step}: {type(error).__name__}: {error}"
|
||||
|
||||
def context_for(self, program_id: str, prompt: str, request_kwargs: Mapping[str, object]) -> ProgramContext:
|
||||
return ProgramContext(
|
||||
program_id=program_id,
|
||||
prompt=prompt,
|
||||
request_type=classify_prompt(prompt),
|
||||
message_logging_off=request_is_private(request_kwargs),
|
||||
caller=forwardable_caller(request_kwargs),
|
||||
)
|
||||
|
||||
def binding(self, program_id: str, owner: str | None = None) -> ProgramBinding | None:
|
||||
"""The program's row as seen by ``owner``, the key that started it (``user_api_key_hash``)."""
|
||||
return self._bindings.get(ProgramKey(owner, program_id))
|
||||
|
||||
def bindings_named(self, program_id: str) -> tuple[ProgramBinding, ...]:
|
||||
"""Every key's program with this id, oldest first: an admin may complete another key's program."""
|
||||
matching: Final = (binding for key, binding in self._bindings.items() if key.program_id == program_id)
|
||||
return tuple(sorted(matching, key=lambda binding: binding.bound_at))
|
||||
|
||||
def _evict_expired(self, now: float) -> None:
|
||||
deadline: Final = now - self.config.program_ttl_seconds
|
||||
expired: Final = tuple(key for key, binding in self._bindings.items() if binding.bound_at < deadline)
|
||||
for key in expired:
|
||||
self._bindings.pop(key, None)
|
||||
overflow: Final = len(self._bindings) + 1 - self.config.max_programs # +1: room for the program being bound
|
||||
oldest: Final = tuple(sorted(self._bindings, key=lambda key: self._bindings[key].bound_at))[: max(overflow, 0)]
|
||||
for key in oldest:
|
||||
self._bindings.pop(key, None)
|
||||
self.programs_evicted += len(expired) + len(oldest)
|
||||
|
||||
async def bind(
|
||||
self, program_id: str, prompt: str, api_key_hash: str | None, request_kwargs: Mapping[str, object] = _EMPTY
|
||||
) -> ProgramBinding:
|
||||
"""Bind a new program, or return its existing row.
|
||||
|
||||
A program is private to the key that started it: the same id under another key starts that key's
|
||||
own program instead of riding on, or completing, someone else's. Concurrent first requests of one
|
||||
program (a harness that fans out) wait for the decision in flight instead of each asking the
|
||||
decision maker, so a program is bound exactly once.
|
||||
"""
|
||||
key: Final = ProgramKey(api_key_hash, program_id)
|
||||
in_flight = self._binding_in_flight.get(key) # rebind-ok: re-read after each wait
|
||||
while in_flight is not None:
|
||||
await in_flight.wait()
|
||||
in_flight = self._binding_in_flight.get(key)
|
||||
existing: Final = self._bindings.get(key)
|
||||
if existing is not None:
|
||||
return existing
|
||||
event: Final = asyncio.Event()
|
||||
self._binding_in_flight[key] = event
|
||||
try:
|
||||
return await self._bind(key, prompt, request_kwargs)
|
||||
finally:
|
||||
self._binding_in_flight.pop(key, None)
|
||||
event.set()
|
||||
|
||||
async def _bind(self, key: ProgramKey, prompt: str, request_kwargs: Mapping[str, object]) -> ProgramBinding:
|
||||
now: Final = self._clock()
|
||||
if len(self._bindings) >= min(PROGRAM_SWEEP_THRESHOLD, self.config.max_programs):
|
||||
self._evict_expired(now)
|
||||
context: Final = self.context_for(key.program_id, prompt, request_kwargs)
|
||||
model: Final = await self.decision_maker.select(context)
|
||||
binding: Final = ProgramBinding(
|
||||
program_id=key.program_id,
|
||||
model=model,
|
||||
context=replace(context, caller=_EMPTY),
|
||||
api_key_hash=key.owner,
|
||||
bound_at=now,
|
||||
message_logging_off=context.message_logging_off,
|
||||
)
|
||||
self._bindings[key] = binding
|
||||
self.programs_bound += 1
|
||||
return binding
|
||||
|
||||
async def async_pre_routing_hook(
|
||||
self,
|
||||
model: str,
|
||||
request_kwargs: dict[str, object], # mutable-ok: pre-routing contract
|
||||
messages: list[dict[str, object]] | None = None, # mutable-ok: pre-routing contract
|
||||
input: str | list[object] | None = None, # mutable-ok: pre-routing contract
|
||||
specific_deployment: bool | None = False,
|
||||
) -> PreRoutingHookResponse | None:
|
||||
"""Route the request to its program's model, binding the program on its first request."""
|
||||
program_id: Final = resolve_program_id(request_kwargs, self.config.program_id_key)
|
||||
prompt: Final = first_user_text(program_messages(messages, input, request_kwargs))
|
||||
chosen_model, request_type = await self._decide(program_id, prompt, request_kwargs)
|
||||
_stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model)
|
||||
_stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id)
|
||||
return PreRoutingHookResponse(
|
||||
model=chosen_model,
|
||||
messages=messages,
|
||||
routing_decision=StandardLoggingRoutingDecision(
|
||||
router_model_name=self.router_name,
|
||||
router_type="oracle",
|
||||
routed_model=chosen_model,
|
||||
cause=self._cause,
|
||||
request_type=request_type.value,
|
||||
),
|
||||
)
|
||||
|
||||
async def _decide(
|
||||
self, program_id: str | None, prompt: str, request_kwargs: Mapping[str, object]
|
||||
) -> tuple[str, RequestType]:
|
||||
if program_id is None:
|
||||
self.stateless_requests += 1
|
||||
context: Final = self.context_for("", prompt, request_kwargs)
|
||||
return await self.decision_maker.select(context), context.request_type
|
||||
binding: Final = await self.bind(program_id, prompt, _api_key_hash(request_kwargs), request_kwargs)
|
||||
return binding.model, binding.context.request_type
|
||||
|
||||
def mark_message_logging_off(self, program_id: str, owner: str | None = None) -> None:
|
||||
"""A request of the program opted out of message logging, whether or not it produced an answer: the
|
||||
whole program is private from here on, and its verification runs under that opt-out."""
|
||||
key: Final = ProgramKey(owner, program_id)
|
||||
binding: Final = self._bindings.get(key)
|
||||
if binding is not None and not binding.message_logging_off:
|
||||
self._bindings[key] = replace(binding, message_logging_off=True)
|
||||
|
||||
def observe_request(
|
||||
self, program_id: str, cost: float, response_text: str, messages: object = None, owner: str | None = None
|
||||
) -> None:
|
||||
"""Account one finished LLM request of the program: LiteLLM's spend, the latest answer and transcript."""
|
||||
key: Final = ProgramKey(owner, program_id)
|
||||
binding: Final = self._bindings.get(key)
|
||||
if binding is None:
|
||||
return
|
||||
transcript: Final = _transcript(messages, binding.last_messages)
|
||||
self._bindings[key] = replace(
|
||||
binding,
|
||||
requests=binding.requests + 1,
|
||||
cost=binding.cost + max(cost, 0.0),
|
||||
last_response_text=response_text or binding.last_response_text,
|
||||
last_messages=transcript,
|
||||
)
|
||||
deferred: Final = binding.deferred
|
||||
if deferred is not None: # the harness reported before this response was observed
|
||||
self.complete(program_id, score=deferred.score, cost=deferred.cost, payload=deferred.payload, owner=owner)
|
||||
|
||||
def complete(
|
||||
self,
|
||||
program_id: str,
|
||||
score: float | None = None,
|
||||
cost: float | None = None,
|
||||
payload: Mapping[str, object] = _EMPTY,
|
||||
owner: str | None = None,
|
||||
) -> asyncio.Task[float] | None:
|
||||
"""Release the program now and verify it in the background; the task resolves to the verified score.
|
||||
|
||||
A program none of whose requests has been observed yet (feedback that raced the last response's
|
||||
logging, or a program whose only request failed) keeps its slot and is completed, with these
|
||||
arguments, by its next observed response: there is no answer to verify yet, so nothing is learned.
|
||||
|
||||
Raises ``VerificationBacklogFull`` when ``max_pending_verifications`` verifications are already in
|
||||
flight: the program stays bound, so the caller can complete it again once the backlog drains.
|
||||
"""
|
||||
key: Final = ProgramKey(owner, program_id)
|
||||
binding: Final = self._bindings.get(key)
|
||||
if binding is None:
|
||||
return None
|
||||
if binding.requests == 0:
|
||||
self._bindings[key] = replace(binding, deferred=DeferredCompletion(score=score, cost=cost, payload=payload))
|
||||
return None
|
||||
if len(self._tasks) >= self.config.max_pending_verifications:
|
||||
self.verifications_rejected += 1
|
||||
raise VerificationBacklogFull(
|
||||
f"{len(self._tasks)} verifications are in flight (max_pending_verifications="
|
||||
f"{self.config.max_pending_verifications}); program {program_id!r} stays bound, complete it again later"
|
||||
)
|
||||
del self._bindings[key]
|
||||
outcome: Final = ProgramOutcome(
|
||||
program_id=program_id,
|
||||
model=binding.model,
|
||||
prompt=binding.context.prompt,
|
||||
response_text=binding.last_response_text,
|
||||
cost=cost if cost is not None else binding.cost,
|
||||
messages=binding.last_messages,
|
||||
score=score,
|
||||
payload=payload,
|
||||
message_logging_off=binding.message_logging_off,
|
||||
)
|
||||
self.programs_completed += 1
|
||||
task: Final = asyncio.create_task(self._verify_and_learn(binding, outcome))
|
||||
self._tasks.add(task)
|
||||
task.add_done_callback(self._tasks.discard)
|
||||
return task
|
||||
|
||||
async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float:
|
||||
try:
|
||||
with inherit_message_logging_privacy(outcome.message_logging_off):
|
||||
score: Final = clamp_score(
|
||||
await asyncio.wait_for(
|
||||
self.verifier.verify(outcome), timeout=self.config.verification_timeout_seconds
|
||||
)
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop
|
||||
self.verifications_failed += 1
|
||||
self.note_failure("verification", error)
|
||||
return float("nan")
|
||||
async with self._update_lock:
|
||||
self.decision_maker.update(binding.context, binding.model, score)
|
||||
self._history.append(
|
||||
FeedbackRecord(
|
||||
program_id=binding.program_id,
|
||||
model=binding.model,
|
||||
request_type=binding.context.request_type.value,
|
||||
score=score,
|
||||
cost=outcome.cost,
|
||||
verified_at=self._clock(),
|
||||
)
|
||||
)
|
||||
return score
|
||||
|
||||
async def drain(self) -> None:
|
||||
"""Wait for every in-flight verification."""
|
||||
while self._tasks:
|
||||
await asyncio.gather(*tuple(self._tasks), return_exceptions=True)
|
||||
|
||||
@property
|
||||
def pending_verifications(self) -> int:
|
||||
return len(self._tasks)
|
||||
|
||||
def _accuracy_by_model(self) -> Mapping[str, float]:
|
||||
scores: Final[dict[str, list[float]]] = {} # mutable-ok: grouping accumulator for the snapshot
|
||||
for record in self._history:
|
||||
scores.setdefault(record.model, []).append(record.score)
|
||||
return {model: round(sum(values) / len(values), 4) for model, values in scores.items()}
|
||||
|
||||
async def get_state_snapshot(self) -> Mapping[str, object]:
|
||||
"""In-memory view for the introspection endpoint."""
|
||||
active: Final[dict[str, int]] = {} # mutable-ok: counting accumulator for the snapshot
|
||||
for binding in self._bindings.values():
|
||||
active[binding.model] = active.get(binding.model, 0) + 1
|
||||
return {
|
||||
"router_name": self.router_name,
|
||||
"available_models": list(self.models),
|
||||
"decision_maker": self.decision_maker.snapshot(),
|
||||
"verifier": type(self.verifier).__name__,
|
||||
"active_programs": len(self._bindings),
|
||||
"active_per_model": active,
|
||||
"programs_bound": self.programs_bound,
|
||||
"programs_completed": self.programs_completed,
|
||||
"programs_evicted": self.programs_evicted,
|
||||
"stateless_requests": self.stateless_requests,
|
||||
"pending_verifications": self.pending_verifications,
|
||||
"verifications_failed": self.verifications_failed,
|
||||
"verifications_rejected": self.verifications_rejected,
|
||||
"last_failure": self.last_failure,
|
||||
"recent_accuracy": self._accuracy_by_model(),
|
||||
"recent_feedback": [
|
||||
{
|
||||
"program_id": record.program_id,
|
||||
"model": record.model,
|
||||
"request_type": record.request_type,
|
||||
"score": record.score,
|
||||
"cost": record.cost,
|
||||
}
|
||||
for record in tuple(self._history)[-20:]
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def request_type_of(prompt: str) -> RequestType:
|
||||
return classify_prompt(prompt)
|
||||
235
litellm/router_strategy/oracle_router/verifier.py
Normal file
235
litellm/router_strategy/oracle_router/verifier.py
Normal file
|
|
@ -0,0 +1,235 @@
|
|||
"""Verifiers score a finished program in [0, 1], off the critical path.
|
||||
|
||||
The contract is one coroutine. ORACLE ships three: any configured LiteLLM guardrail applied to the
|
||||
program's final turn with its transcript as context (``llm_as_a_judge``, ``custom_code`` or a hosted
|
||||
check), the score the client reported, and an import path for an operator's own class.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol, runtime_checkable
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, REDACTED_BY_LITELLM
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import OracleVerifierConfig
|
||||
from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse
|
||||
|
||||
from .decision import load_object
|
||||
|
||||
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
|
||||
_STATUS_RECORDS: Final = TypeAdapter(Sequence[Mapping[str, object]])
|
||||
_STATUS_RECORD_KEY: Final[str] = "standard_logging_guardrail_information"
|
||||
_PROXY_OWNED_METADATA_KEYS: Final[frozenset[str]] = frozenset(
|
||||
{_STATUS_RECORD_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "headers", "routing_decision"}
|
||||
)
|
||||
_BLOCKED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_intervened", "guardrail_flagged"})
|
||||
_UNVERIFIED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_failed_to_respond", "not_run"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProgramOutcome:
|
||||
"""Everything a verifier may look at once a program has finished."""
|
||||
|
||||
program_id: str
|
||||
model: str
|
||||
prompt: str
|
||||
response_text: str
|
||||
cost: float
|
||||
messages: Sequence[AllMessageValues] = ()
|
||||
score: float | None = None
|
||||
payload: Mapping[str, object] = field(default_factory=lambda: _EMPTY)
|
||||
message_logging_off: bool = False
|
||||
"""Whether any request of the program opted out of message logging; the router verifies under that opt-out."""
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Verifier(Protocol):
|
||||
async def verify(self, outcome: ProgramOutcome) -> float: ...
|
||||
|
||||
|
||||
def clamp_score(value: float) -> float:
|
||||
"""Clamp a score to [0, 1]. A NaN/inf score raises: it would corrupt a bandit cell for good."""
|
||||
score: Final = float(value)
|
||||
if not math.isfinite(score):
|
||||
raise ValueError(f"score must be finite, got {score!r}")
|
||||
return min(max(score, 0.0), 1.0)
|
||||
|
||||
|
||||
def response_text(response: object) -> str:
|
||||
"""The assistant text of a chat completion, or an empty string for anything else."""
|
||||
if not isinstance(response, ModelResponse) or not response.choices:
|
||||
return ""
|
||||
content: Final = response.choices[0].message.content
|
||||
return content if isinstance(content, str) else ""
|
||||
|
||||
|
||||
class ReportedVerifier:
|
||||
"""Trust the score the client reported with the program's last request or through the feedback endpoint."""
|
||||
|
||||
def __init__(self, default: float = 0.0) -> None:
|
||||
self._default: Final[float] = default
|
||||
|
||||
async def verify(self, outcome: ProgramOutcome) -> float:
|
||||
return clamp_score(outcome.score) if outcome.score is not None else self._default
|
||||
|
||||
|
||||
def find_guardrail(name: str) -> CustomGuardrail | None:
|
||||
"""The initialized guardrail registered under ``name``, if any."""
|
||||
callbacks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(callback_type=CustomGuardrail)
|
||||
guardrails: Final = tuple(callback for callback in callbacks if isinstance(callback, CustomGuardrail))
|
||||
return next((guardrail for guardrail in guardrails if guardrail.guardrail_name == name), None)
|
||||
|
||||
|
||||
def _redacted(outcome: ProgramOutcome) -> bool:
|
||||
"""Whether LiteLLM replaced the program's final answer or transcript with its redaction placeholder."""
|
||||
return outcome.response_text == REDACTED_BY_LITELLM or any(
|
||||
_STR_MAPPING.validate_python(message).get("content") == REDACTED_BY_LITELLM for message in outcome.messages
|
||||
)
|
||||
|
||||
|
||||
def _verifier_metadata(outcome: ProgramOutcome) -> dict[str, object]: # mutable-ok: the guardrail writes into it
|
||||
"""The feedback payload as the guardrail's request metadata.
|
||||
|
||||
The verdict is read from the status record the guardrail writes here, so the caller's payload must not
|
||||
pre-seed that key (a non-record entry would make the whole list unreadable and hide a rejection), it
|
||||
cannot rename the program, and it cannot pose as the proxy: the ``user_api_key_*`` identity a guardrail
|
||||
may forward to its vendor, the sub-call origin and the request headers are the proxy's to stamp.
|
||||
"""
|
||||
return {
|
||||
**{key: value for key, value in outcome.payload.items() if not _proxy_owned(key)},
|
||||
"program_id": outcome.program_id,
|
||||
}
|
||||
|
||||
|
||||
def _proxy_owned(key: str) -> bool:
|
||||
return key.startswith("user_api_key") or key in _PROXY_OWNED_METADATA_KEYS
|
||||
|
||||
|
||||
def _status_records(bucket: object) -> Sequence[Mapping[str, object]]:
|
||||
"""The ``standard_logging_guardrail_information`` records of one metadata bucket, or none."""
|
||||
try:
|
||||
return _STATUS_RECORDS.validate_python(_STR_MAPPING.validate_python(bucket).get(_STATUS_RECORD_KEY) or ())
|
||||
except ValidationError:
|
||||
return ()
|
||||
|
||||
|
||||
def recorded_status(request_data: Mapping[str, object], guardrail_name: str) -> str | None:
|
||||
"""The last status ``guardrail_name`` recorded for itself in ``request_data`` while it ran, if any.
|
||||
|
||||
Every first-party guardrail writes a ``standard_logging_guardrail_information`` record into the
|
||||
request's metadata bucket (``success``, ``guardrail_intervened``, ``guardrail_failed_to_respond``,
|
||||
...), which is the only way to tell a judge that failed the answer from one that failed to answer.
|
||||
"""
|
||||
records: Final = (
|
||||
*_status_records(request_data.get("metadata")),
|
||||
*_status_records(request_data.get("litellm_metadata")),
|
||||
)
|
||||
statuses: Final = tuple(
|
||||
str(record["guardrail_status"])
|
||||
for record in records
|
||||
if record.get("guardrail_name") in (guardrail_name, None) and record.get("guardrail_status")
|
||||
)
|
||||
return statuses[-1] if statuses else None
|
||||
|
||||
|
||||
async def _apply(
|
||||
guardrail: CustomGuardrail,
|
||||
inputs: GenericGuardrailAPIInputs,
|
||||
request_data: dict[str, object], # mutable-ok: the guardrail writes its status into it
|
||||
) -> Exception | None:
|
||||
"""Run the guardrail; the exception it raised to signal its verdict, or ``None`` when it returned."""
|
||||
try:
|
||||
await guardrail.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # upstream types request_data as a bare dict
|
||||
inputs=inputs, request_data=request_data, input_type="response"
|
||||
)
|
||||
except Exception as error: # noqa: BLE001 # a guardrail may signal its verdict by raising; the caller classifies it
|
||||
return error
|
||||
return None
|
||||
|
||||
|
||||
def _is_http_block(error: Exception) -> bool:
|
||||
"""A guardrail without a status record that blocks by raising FastAPI's ``HTTPException``."""
|
||||
try:
|
||||
from fastapi import HTTPException
|
||||
except ImportError: # the SDK without the proxy extra: nothing can raise it
|
||||
return False
|
||||
return isinstance(error, HTTPException)
|
||||
|
||||
|
||||
class GuardrailVerifier:
|
||||
"""Apply a configured LiteLLM guardrail to the program's final turn: it passes or it is blocked.
|
||||
|
||||
The guardrail sees the final assistant text as ``texts``, the program's transcript as
|
||||
``structured_messages`` and the feedback payload as ``request_data["metadata"]``, so an
|
||||
``llm_as_a_judge`` guardrail judges the whole conversation and a ``custom_code`` guardrail can read
|
||||
whatever the harness posted with the feedback.
|
||||
|
||||
The verdict is read from the status the guardrail records for itself (so a judge configured with
|
||||
``on_failure: log`` still scores 0 when the answer fails), and falls back to how it raised: a
|
||||
``GuardrailRaisedException`` with ``blocked_content`` or an ``HTTPException`` is a block. A
|
||||
guardrail that could not evaluate the program (``guardrail_failed_to_respond``, an unreachable
|
||||
backend) raises, so the program counts as a failed verification and teaches the decision maker
|
||||
nothing instead of passing it.
|
||||
"""
|
||||
|
||||
def __init__(self, guardrail_name: str, lookup: Callable[[str], CustomGuardrail | None] = find_guardrail) -> None:
|
||||
self._name: Final[str] = guardrail_name
|
||||
self._lookup: Final = lookup
|
||||
|
||||
async def verify(self, outcome: ProgramOutcome) -> float:
|
||||
guardrail: Final = self._lookup(self._name)
|
||||
if guardrail is None:
|
||||
raise LookupError(f"guardrail {self._name!r} is not initialized on this proxy")
|
||||
if not outcome.response_text.strip():
|
||||
# A judge skips an empty answer without recording a verdict; that must not read as a pass.
|
||||
raise RuntimeError(f"guardrail {self._name!r}: the program has no response text to verify")
|
||||
if _redacted(outcome):
|
||||
# The request opted out of message logging, so the router only ever saw LiteLLM's placeholder.
|
||||
raise RuntimeError(f"guardrail {self._name!r}: the program's final turn was redacted before verification")
|
||||
transcript: Final = list(outcome.messages) or [{"role": "user", "content": outcome.prompt}]
|
||||
inputs: Final = GenericGuardrailAPIInputs(
|
||||
texts=[outcome.response_text], model=outcome.model, structured_messages=transcript
|
||||
)
|
||||
request_data: Final[dict[str, object]] = { # mutable-ok: the guardrail records its status into it
|
||||
"model": outcome.model,
|
||||
"metadata": _verifier_metadata(outcome),
|
||||
}
|
||||
verdict_error: Final = await _apply(guardrail, inputs, request_data)
|
||||
status: Final = recorded_status(request_data, self._name)
|
||||
if status in _UNVERIFIED_STATUSES:
|
||||
raise RuntimeError(f"guardrail {self._name!r} did not evaluate program {outcome.program_id!r}: {status}")
|
||||
if status in _BLOCKED_STATUSES:
|
||||
return 0.0
|
||||
if status == "success" or verdict_error is None:
|
||||
return 1.0
|
||||
if isinstance(verdict_error, GuardrailRaisedException):
|
||||
if verdict_error.blocked_content:
|
||||
return 0.0
|
||||
raise verdict_error # the guardrail could not reach a verdict
|
||||
if _is_http_block(verdict_error):
|
||||
return 0.0
|
||||
raise verdict_error
|
||||
|
||||
|
||||
def build_verifier(config: OracleVerifierConfig) -> Verifier:
|
||||
match config.type:
|
||||
case "reported":
|
||||
return ReportedVerifier(default=config.default_score)
|
||||
case "guardrail":
|
||||
return GuardrailVerifier(config.guardrail or "")
|
||||
case "custom":
|
||||
factory: Final = load_object(config.path or "")
|
||||
if not callable(factory):
|
||||
raise TypeError(f"oracle_router verifier.path {config.path!r} is not callable")
|
||||
instance: Final = factory()
|
||||
if not isinstance(instance, Verifier):
|
||||
raise TypeError(f"{config.path!r} must build an object with an async verify(outcome) method")
|
||||
return instance
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
"""Types for the ORACLE router management endpoints."""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class OracleRouterFeedbackRequest(BaseModel, frozen=True):
|
||||
"""Report that a program finished. ``score`` is optional when the router's verifier grades it."""
|
||||
|
||||
program_id: str = Field(min_length=1)
|
||||
score: float | None = Field(default=None, ge=0.0, le=1.0)
|
||||
cost: float | None = Field(
|
||||
default=None, ge=0.0, description="Overrides the spend LiteLLM accumulated for the program"
|
||||
)
|
||||
payload: Mapping[str, object] = Field(default_factory=dict, description="Passed to the verifier unchanged")
|
||||
|
||||
|
||||
class OracleRouterFeedbackResponse(BaseModel, frozen=True):
|
||||
program_id: str
|
||||
router_name: str
|
||||
model: str
|
||||
pending_verifications: int
|
||||
deferred: bool = Field(
|
||||
default=False,
|
||||
description="No response of the program has been observed yet; its next one completes it with this feedback",
|
||||
)
|
||||
|
||||
|
||||
class OracleRouterStateResponse(BaseModel, frozen=True):
|
||||
routers: Sequence[Mapping[str, object]]
|
||||
|
|
@ -675,6 +675,8 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
|
|||
# adaptive-router params
|
||||
adaptive_router_default_model: str | None = None
|
||||
adaptive_router_config: dict | None = None
|
||||
# oracle-router params
|
||||
oracle_router_config: Mapping[str, object] | None = None
|
||||
# quality-router params
|
||||
quality_router_config: dict | None = None
|
||||
quality_router_default_model: str | None = None
|
||||
|
|
@ -1531,6 +1533,82 @@ class AdaptiveRouterPreferences(LiteLLMBaseModel):
|
|||
strengths: list[RequestType] = Field(default_factory=list)
|
||||
|
||||
|
||||
OracleDecisionMakerType: TypeAlias = Literal["thompson", "fixed", "pre_routing", "custom"]
|
||||
OracleVerifierType: TypeAlias = Literal["reported", "guardrail", "custom"]
|
||||
|
||||
|
||||
class OracleDecisionMakerConfig(BaseModel, frozen=True):
|
||||
"""Which policy binds a program to a model, and its knobs.
|
||||
|
||||
``thompson`` (default) is the adaptive router's bandit trained on verified program outcomes; ``weights``
|
||||
are that bandit's quality/cost weights. ``fixed`` always picks ``model``. ``pre_routing`` lets the strategy
|
||||
router named by ``router`` (a complexity, quality, adaptive or semantic router in the same model_list)
|
||||
propose the model once per program. ``custom`` imports ``path`` (``package.module:Factory``) and calls it
|
||||
with the model tuple; it is honoured only for deployments in the proxy config file, never for models
|
||||
written through the API.
|
||||
"""
|
||||
|
||||
type: OracleDecisionMakerType = "thompson"
|
||||
weights: AdaptiveRouterWeights = Field(default_factory=AdaptiveRouterWeights)
|
||||
seed: int = 0
|
||||
model: str | None = None
|
||||
router: str | None = None
|
||||
path: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _required_fields(self) -> "OracleDecisionMakerConfig":
|
||||
required: Final = {"fixed": self.model, "pre_routing": self.router, "custom": self.path}
|
||||
if self.type in required and not required[self.type]:
|
||||
names: Final = {"fixed": "model", "pre_routing": "router", "custom": "path"}
|
||||
raise ValueError(f"decision_maker.type={self.type!r} requires decision_maker.{names[self.type]}")
|
||||
return self
|
||||
|
||||
|
||||
class OracleVerifierConfig(BaseModel, frozen=True):
|
||||
"""How a finished program is scored.
|
||||
|
||||
``guardrail`` applies the configured LiteLLM guardrail ``guardrail`` (for example an ``llm_as_a_judge``
|
||||
guardrail) to the program's final turn with the transcript as context. ``reported`` trusts
|
||||
``metadata.program_score`` or the feedback endpoint. ``custom`` imports ``path`` and calls it with no
|
||||
arguments; it is honoured only for deployments in the proxy config file, never for models written
|
||||
through the API.
|
||||
"""
|
||||
|
||||
type: OracleVerifierType = "reported"
|
||||
default_score: float = Field(default=0.0, ge=0.0, le=1.0)
|
||||
guardrail: str | None = None
|
||||
path: str | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _required_fields(self) -> "OracleVerifierConfig":
|
||||
required: Final = {"guardrail": self.guardrail, "custom": self.path}
|
||||
if self.type in required and not required[self.type]:
|
||||
names: Final = {"guardrail": "guardrail", "custom": "path"}
|
||||
raise ValueError(f"verifier.type={self.type!r} requires verifier.{names[self.type]}")
|
||||
return self
|
||||
|
||||
|
||||
class OracleRouterConfig(BaseModel, frozen=True):
|
||||
"""``litellm_params.oracle_router_config`` of an ``auto_router/oracle_router`` deployment."""
|
||||
|
||||
available_models: tuple[str, ...] = Field(min_length=1)
|
||||
decision_maker: OracleDecisionMakerConfig = Field(default_factory=OracleDecisionMakerConfig)
|
||||
verifier: OracleVerifierConfig = Field(default_factory=OracleVerifierConfig)
|
||||
program_id_key: str = "program_id"
|
||||
program_ttl_seconds: int = Field(default=24 * 3600, ge=1)
|
||||
max_programs: int = Field(default=10_000, ge=1)
|
||||
max_pending_verifications: int = Field(default=64, ge=1)
|
||||
verification_timeout_seconds: float = Field(default=120.0, gt=0)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _fixed_model_is_available(self) -> "OracleRouterConfig":
|
||||
if len(self.available_models) != len(frozenset(self.available_models)):
|
||||
raise ValueError("available_models must be distinct")
|
||||
if self.decision_maker.type == "fixed" and self.decision_maker.model not in self.available_models:
|
||||
raise ValueError("decision_maker.model must be one of available_models")
|
||||
return self
|
||||
|
||||
|
||||
def reject_server_owned_wif_params(body: Mapping[str, object]) -> None:
|
||||
"""Raise ``ValueError`` if a mapping that did not come from deployment config carries a
|
||||
server-owned workload identity federation or OAuth token-exchange field, or an ``api_key``
|
||||
|
|
|
|||
|
|
@ -3374,7 +3374,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
|
|||
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""
|
||||
|
||||
router_model_name: str
|
||||
router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]]
|
||||
router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic", "oracle"]]
|
||||
router_config_id: ReadOnly[str]
|
||||
router_config_updated_at: ReadOnly[str]
|
||||
router_config_fingerprint: ReadOnly[str]
|
||||
|
|
|
|||
|
|
@ -1085,7 +1085,7 @@ class TestClearCache:
|
|||
Test that clear_cache successfully clears router model caches and reloads models.
|
||||
"""
|
||||
mock_router = MagicMock()
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
|
||||
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
|
||||
mock_router.registry.return_value.all.return_value = ["openai/gpt-4o", "openai/gpt-4o-mini"]
|
||||
|
||||
|
|
@ -1125,7 +1125,7 @@ class TestClearCache:
|
|||
# entries are auto_router/* deployments (so their router-map entries should be
|
||||
# cleared for reload); the config-defined router is preserved.
|
||||
mock_router = MagicMock()
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
|
||||
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
|
||||
mock_router.registry.return_value.all.return_value = [
|
||||
{
|
||||
|
|
@ -1209,7 +1209,7 @@ class TestClearCache:
|
|||
)
|
||||
|
||||
mock_router = MagicMock()
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
|
||||
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
|
||||
mock_router.registry.return_value.all.return_value = [
|
||||
{
|
||||
|
|
@ -1270,7 +1270,7 @@ class TestClearCachePreservesConfigRouters:
|
|||
)
|
||||
|
||||
mock_router = MagicMock()
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
|
||||
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
|
||||
mock_router.registry.return_value.all.return_value = [
|
||||
{
|
||||
|
|
@ -1315,7 +1315,7 @@ class TestClearCachePreservesConfigRouters:
|
|||
from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache
|
||||
|
||||
mock_router = MagicMock()
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
|
||||
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
|
||||
mock_router.registry.return_value.all.return_value = [
|
||||
{
|
||||
|
|
@ -1354,7 +1354,7 @@ class TestClearCachePreservesConfigRouters:
|
|||
from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache
|
||||
|
||||
mock_router = MagicMock()
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
|
||||
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
|
||||
mock_router.registry.return_value.all.return_value = [
|
||||
{
|
||||
|
|
@ -1367,11 +1367,17 @@ class TestClearCachePreservesConfigRouters:
|
|||
"model_info": {"id": "db-a", "db_model": True},
|
||||
"litellm_params": {"model": "auto_router/adaptive_router/a1"},
|
||||
},
|
||||
{
|
||||
"model_name": "o1",
|
||||
"model_info": {"id": "db-o", "db_model": True},
|
||||
"litellm_params": {"model": "auto_router/oracle_router/o1"},
|
||||
},
|
||||
]
|
||||
plugins["auto"] = {}
|
||||
plugins["complexity"] = {}
|
||||
plugins["quality"] = {"q1": MagicMock()}
|
||||
plugins["adaptive"] = {"a1": MagicMock()}
|
||||
plugins["oracle"] = {"o1": MagicMock()}
|
||||
|
||||
mock_config = MagicMock()
|
||||
mock_config.add_deployment_locked = AsyncMock(
|
||||
|
|
@ -1389,6 +1395,7 @@ class TestClearCachePreservesConfigRouters:
|
|||
|
||||
assert "q1" not in plugins["quality"]
|
||||
assert "a1" not in plugins["adaptive"]
|
||||
assert "o1" not in plugins["oracle"]
|
||||
|
||||
|
||||
class TestDeleteModelClearsRouterRegistry:
|
||||
|
|
@ -1503,7 +1510,7 @@ class TestDeleteModelClearsRouterRegistry:
|
|||
mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row)
|
||||
|
||||
mock_router = MagicMock()
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
|
||||
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
|
||||
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
|
||||
mock_router.apply.return_value = ApplyResult(
|
||||
version=1,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,190 @@
|
|||
"""POST /oracle_router/feedback and GET /oracle_router/state."""
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.oracle_router_endpoints import (
|
||||
get_oracle_router_state,
|
||||
submit_oracle_router_feedback,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
|
||||
from litellm.router_strategy.oracle_router.verifier import ReportedVerifier
|
||||
from litellm.types.management_endpoints.oracle_router_endpoints import OracleRouterFeedbackRequest
|
||||
from litellm.types.router import OracleRouterConfig, TaggedPreRoutingStrategy
|
||||
|
||||
ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="admin-hash")
|
||||
OWNER = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="owner-hash")
|
||||
STRANGER = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="other-hash")
|
||||
|
||||
|
||||
class _Maker:
|
||||
def __init__(self) -> None:
|
||||
self.updates = []
|
||||
|
||||
@property
|
||||
def models(self):
|
||||
return ("smart", "fast")
|
||||
|
||||
async def select(self, context):
|
||||
return "smart"
|
||||
|
||||
def update(self, context, model, score):
|
||||
self.updates.append((model, score))
|
||||
|
||||
def snapshot(self):
|
||||
return {"type": "test"}
|
||||
|
||||
|
||||
class _Registry:
|
||||
def __init__(self, oracle: OracleRouter) -> None:
|
||||
self._oracle_routers = {oracle.router_name: [TaggedPreRoutingStrategy(tags=(), strategy=oracle)]}
|
||||
|
||||
def plugins(self, kind: str):
|
||||
assert kind == "oracle"
|
||||
return self._oracle_routers
|
||||
|
||||
|
||||
class _LLMRouter:
|
||||
def __init__(self, oracle: OracleRouter) -> None:
|
||||
self._registry = _Registry(oracle)
|
||||
|
||||
def registry(self) -> _Registry:
|
||||
return self._registry
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def oracle():
|
||||
maker = _Maker()
|
||||
router = OracleRouter("oracle", OracleRouterConfig(available_models=["smart", "fast"]), maker, ReportedVerifier())
|
||||
router.maker = maker
|
||||
return router
|
||||
|
||||
|
||||
async def _bind(oracle: OracleRouter, program_id: str, api_key_hash: str | None = "owner-hash"):
|
||||
metadata = {"program_id": program_id, **({"user_api_key_hash": api_key_hash} if api_key_hash else {})}
|
||||
await oracle.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": metadata}, messages=[{"role": "user", "content": "t"}]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_completes_the_program_and_the_router_learns(oracle):
|
||||
await _bind(oracle, "task-1")
|
||||
oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash")
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
|
||||
response = await submit_oracle_router_feedback(
|
||||
OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER
|
||||
)
|
||||
assert response.model_dump() == {
|
||||
"program_id": "task-1",
|
||||
"router_name": "oracle",
|
||||
"model": "smart",
|
||||
"pending_verifications": 1,
|
||||
"deferred": False,
|
||||
}
|
||||
assert oracle.binding("task-1", "owner-hash") is None
|
||||
await oracle.drain()
|
||||
assert oracle.maker.updates == [("smart", 1.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_is_rejected_with_503_while_the_verification_backlog_is_full():
|
||||
class _Stuck:
|
||||
async def verify(self, outcome):
|
||||
await asyncio.sleep(5.0)
|
||||
return 1.0
|
||||
|
||||
config = OracleRouterConfig(available_models=["smart", "fast"], max_pending_verifications=1)
|
||||
oracle = OracleRouter("oracle", config, _Maker(), _Stuck())
|
||||
for program_id in ("task-1", "task-2"):
|
||||
await _bind(oracle, program_id)
|
||||
oracle.observe_request(program_id, cost=0.2, response_text="done", owner="owner-hash")
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
|
||||
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER)
|
||||
with pytest.raises(HTTPException) as rejected:
|
||||
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-2", score=1.0), OWNER)
|
||||
assert rejected.value.status_code == 503 and "max_pending_verifications=1" in rejected.value.detail["error"]
|
||||
assert oracle.binding("task-2", "owner-hash") is not None
|
||||
for task in tuple(oracle._tasks):
|
||||
task.cancel()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it(oracle):
|
||||
await _bind(oracle, "task-1")
|
||||
oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash")
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
|
||||
with pytest.raises(HTTPException) as unseen:
|
||||
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), STRANGER)
|
||||
assert unseen.value.status_code == 404 and oracle.binding("task-1", "owner-hash") is not None
|
||||
response = await submit_oracle_router_feedback(
|
||||
OracleRouterFeedbackRequest(program_id="task-1", score=0.0), ADMIN
|
||||
)
|
||||
assert response.program_id == "task-1" and oracle.binding("task-1", "owner-hash") is None
|
||||
await oracle.drain()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across_keys(oracle):
|
||||
await _bind(oracle, "task-1", "owner-hash")
|
||||
await _bind(oracle, "task-1", "other-hash")
|
||||
oracle.observe_request("task-1", cost=0.2, response_text="done", owner="other-hash")
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
|
||||
response = await submit_oracle_router_feedback(
|
||||
OracleRouterFeedbackRequest(program_id="task-1", score=0.5), STRANGER
|
||||
)
|
||||
assert response.program_id == "task-1"
|
||||
assert oracle.binding("task-1", "other-hash") is None and oracle.binding("task-1", "owner-hash") is not None
|
||||
await oracle.drain()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_before_any_response_is_deferred_until_one_is_observed(oracle):
|
||||
await _bind(oracle, "task-1") # bound, but its first response has not been observed yet
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
|
||||
response = await submit_oracle_router_feedback(
|
||||
OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER
|
||||
)
|
||||
assert response.deferred is True and response.pending_verifications == 0
|
||||
assert oracle.binding("task-1", "owner-hash") is not None and oracle.maker.updates == []
|
||||
oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash")
|
||||
assert oracle.binding("task-1", "owner-hash") is None
|
||||
await oracle.drain()
|
||||
assert oracle.maker.updates == [("smart", 1.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_for_an_unknown_program_or_without_routers_is_404(oracle):
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
|
||||
with pytest.raises(HTTPException) as unknown:
|
||||
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="nope"), OWNER)
|
||||
assert unknown.value.status_code == 404
|
||||
with patch("litellm.proxy.proxy_server.llm_router", None):
|
||||
with pytest.raises(HTTPException) as none:
|
||||
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="nope"), OWNER)
|
||||
assert none.value.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_state_is_admin_only_and_lists_every_router(oracle):
|
||||
await _bind(oracle, "task-1")
|
||||
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
|
||||
with pytest.raises(HTTPException) as denied:
|
||||
await get_oracle_router_state(OWNER)
|
||||
assert denied.value.status_code == 403
|
||||
state = await get_oracle_router_state(ADMIN)
|
||||
assert len(state.routers) == 1
|
||||
assert state.routers[0]["router_name"] == "oracle" and state.routers[0]["active_programs"] == 1
|
||||
assert state.routers[0]["decision_maker"] == {"type": "test"}
|
||||
|
||||
|
||||
def test_feedback_request_validates_score_range():
|
||||
with pytest.raises(ValidationError, match="score"):
|
||||
OracleRouterFeedbackRequest(program_id="p", score=1.5)
|
||||
with pytest.raises(ValidationError, match="program_id"):
|
||||
OracleRouterFeedbackRequest(program_id="")
|
||||
0
tests/unit/router_strategy/oracle_router/__init__.py
Normal file
0
tests/unit/router_strategy/oracle_router/__init__.py
Normal file
210
tests/unit/router_strategy/oracle_router/test_decision.py
Normal file
210
tests/unit/router_strategy/oracle_router/test_decision.py
Normal file
|
|
@ -0,0 +1,210 @@
|
|||
"""Decision makers: each learns or proposes the way its contract says."""
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.internal_call_metadata import effective_turn_off_message_logging
|
||||
from litellm.router_strategy.oracle_router.decision import (
|
||||
DecisionMaker,
|
||||
FixedDecisionMaker,
|
||||
PreRoutingDecisionMaker,
|
||||
ProgramContext,
|
||||
ThompsonDecisionMaker,
|
||||
build_decision_maker,
|
||||
load_object,
|
||||
)
|
||||
from litellm.types.router import AdaptiveRouterWeights, OracleDecisionMakerConfig, PreRoutingHookResponse, RequestType
|
||||
|
||||
MODELS = ("smart", "fast")
|
||||
COSTS = {"smart": 1e-5, "fast": 1e-6}
|
||||
|
||||
|
||||
def _ctx(prompt: str = "fix the failing test", request_type: RequestType = RequestType.GENERAL) -> ProgramContext:
|
||||
return ProgramContext(program_id="p", prompt=prompt, request_type=request_type)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thompson_learns_a_different_winner_per_request_type():
|
||||
maker = ThompsonDecisionMaker(MODELS, COSTS, weights=AdaptiveRouterWeights(quality=1.0, cost=0.0), seed=0)
|
||||
for _ in range(80):
|
||||
for request_type, good in ((RequestType.CODE_GENERATION, "smart"), (RequestType.WRITING, "fast")):
|
||||
ctx = _ctx(request_type=request_type)
|
||||
chosen = await maker.select(ctx)
|
||||
maker.update(ctx, chosen, 1.0 if chosen == good else 0.0)
|
||||
code, writing = _ctx(request_type=RequestType.CODE_GENERATION), _ctx(request_type=RequestType.WRITING)
|
||||
assert maker.estimate(code, "smart") > maker.estimate(code, "fast")
|
||||
assert maker.estimate(writing, "fast") > maker.estimate(writing, "smart")
|
||||
picks = [await maker.select(code) for _ in range(30)]
|
||||
assert picks.count("smart") > 24
|
||||
assert maker.snapshot()["cells"]["code_generation/smart"]["samples"] > 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thompson_cost_weight_prefers_the_cheap_model_when_quality_ties():
|
||||
maker = ThompsonDecisionMaker(MODELS, COSTS, weights=AdaptiveRouterWeights(quality=0.3, cost=0.7), seed=1)
|
||||
ctx = _ctx()
|
||||
for _ in range(40):
|
||||
maker.update(ctx, "smart", 0.8)
|
||||
maker.update(ctx, "fast", 0.8)
|
||||
picks = [await maker.select(ctx) for _ in range(30)]
|
||||
assert picks.count("fast") > 24
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_thompson_priors_come_from_adaptive_router_preferences():
|
||||
from litellm.types.router import AdaptiveRouterPreferences
|
||||
|
||||
prefs = {
|
||||
"smart": AdaptiveRouterPreferences(quality_tier=3, strengths=[]),
|
||||
"fast": AdaptiveRouterPreferences(quality_tier=1, strengths=[]),
|
||||
}
|
||||
maker = ThompsonDecisionMaker(
|
||||
MODELS, COSTS, weights=AdaptiveRouterWeights(quality=1.0, cost=0.0), model_prefs=prefs, seed=0
|
||||
)
|
||||
ctx = _ctx()
|
||||
assert maker.estimate(ctx, "smart") > maker.estimate(ctx, "fast")
|
||||
picks = [await maker.select(ctx) for _ in range(30)]
|
||||
assert picks.count("smart") > 24
|
||||
|
||||
|
||||
def test_thompson_rejects_unknown_model_and_empty_pool():
|
||||
maker = ThompsonDecisionMaker(MODELS, COSTS)
|
||||
with pytest.raises(KeyError):
|
||||
maker.update(_ctx(), "other", 1.0)
|
||||
with pytest.raises(ValueError, match="at least one model"):
|
||||
ThompsonDecisionMaker((), COSTS)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fixed_always_returns_its_model_and_counts_updates():
|
||||
maker = FixedDecisionMaker(MODELS, model="fast")
|
||||
assert await maker.select(_ctx()) == "fast"
|
||||
maker.update(_ctx(), "fast", 1.0)
|
||||
assert maker.snapshot() == {"type": "fixed", "model": "fast", "updates": 1}
|
||||
with pytest.raises(KeyError):
|
||||
FixedDecisionMaker(MODELS, model="nope")
|
||||
|
||||
|
||||
class _FakeStrategy:
|
||||
def __init__(self, answer: str | None) -> None:
|
||||
self.answer = answer
|
||||
self.calls: list[dict] = []
|
||||
|
||||
async def async_pre_routing_hook(self, model, request_kwargs, messages=None, input=None, specific_deployment=False):
|
||||
self.calls.append({"model": model, "messages": messages})
|
||||
self.request_kwargs = request_kwargs
|
||||
self.privacy = (
|
||||
effective_turn_off_message_logging(request_kwargs),
|
||||
initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging"),
|
||||
)
|
||||
return None if self.answer is None else PreRoutingHookResponse(model=self.answer, messages=messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_routing_uses_the_wrapped_strategy_proposal_once_per_program():
|
||||
strategy = _FakeStrategy("fast")
|
||||
maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy)
|
||||
assert await maker.select(_ctx("write a haiku")) == "fast"
|
||||
assert strategy.calls == [{"model": "tiered", "messages": [{"role": "user", "content": "write a haiku"}]}]
|
||||
assert maker.snapshot()["proposals"] == {"fast": 1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_routing_forwards_the_program_s_message_logging_opt_out_to_the_wrapped_strategy():
|
||||
strategy = _FakeStrategy("fast")
|
||||
maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy)
|
||||
private = ProgramContext(
|
||||
program_id="p", prompt="secret", request_type=RequestType.GENERAL, message_logging_off=True
|
||||
)
|
||||
assert await maker.select(private) == "fast"
|
||||
assert strategy.privacy == (True, True)
|
||||
assert await maker.select(_ctx("public")) == "fast"
|
||||
assert strategy.privacy == (None, None)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_routing_forwards_the_caller_s_identity_so_a_classifier_call_is_billed_to_the_caller():
|
||||
strategy = _FakeStrategy("fast")
|
||||
maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy)
|
||||
caller = {"user_api_key_hash": "hash-1", "user_api_key_team_id": "team-1", "user_api_key_user_id": "user-1"}
|
||||
private = ProgramContext(
|
||||
program_id="p", prompt="task", request_type=RequestType.GENERAL, message_logging_off=True, caller=caller
|
||||
)
|
||||
assert await maker.select(private) == "fast"
|
||||
assert strategy.request_kwargs == {"metadata": {**caller, "turn_off_message_logging": True}}
|
||||
first_call = strategy.request_kwargs
|
||||
assert await maker.select(private) == "fast"
|
||||
assert strategy.request_kwargs == first_call and strategy.request_kwargs is not first_call
|
||||
assert await maker.select(_ctx()) == "fast"
|
||||
assert strategy.request_kwargs == {"metadata": {}}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_routing_falls_back_to_the_first_model_when_the_proposal_is_unusable():
|
||||
assert await PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy(None)).select(_ctx()) == "smart"
|
||||
maker = PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy("not-configured"))
|
||||
assert await maker.select(_ctx()) == "smart"
|
||||
assert maker.snapshot()["fallbacks"] == 1
|
||||
|
||||
|
||||
class CustomMaker:
|
||||
def __init__(self, models: tuple[str, ...]) -> None:
|
||||
self._models = models
|
||||
|
||||
@property
|
||||
def models(self) -> tuple[str, ...]:
|
||||
return self._models
|
||||
|
||||
async def select(self, context: ProgramContext) -> str:
|
||||
return self._models[-1]
|
||||
|
||||
def update(self, context: ProgramContext, model: str, score: float) -> None:
|
||||
pass
|
||||
|
||||
def snapshot(self):
|
||||
return {"type": "custom"}
|
||||
|
||||
|
||||
class NotAMaker:
|
||||
def __init__(self, models: tuple[str, ...]) -> None:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_decision_maker_by_type():
|
||||
resolve = {"tiered": _FakeStrategy("smart")}.get
|
||||
thompson = build_decision_maker(OracleDecisionMakerConfig(), MODELS, COSTS, resolve)
|
||||
assert isinstance(thompson, ThompsonDecisionMaker) and await thompson.select(_ctx()) in MODELS
|
||||
assert isinstance(
|
||||
build_decision_maker(OracleDecisionMakerConfig(type="fixed", model="fast"), MODELS, COSTS, resolve),
|
||||
FixedDecisionMaker,
|
||||
)
|
||||
pre = build_decision_maker(OracleDecisionMakerConfig(type="pre_routing", router="tiered"), MODELS, COSTS, resolve)
|
||||
assert isinstance(pre, PreRoutingDecisionMaker) and await pre.select(_ctx()) == "smart"
|
||||
with pytest.raises(ValueError, match="not a configured strategy router"):
|
||||
build_decision_maker(OracleDecisionMakerConfig(type="pre_routing", router="missing"), MODELS, COSTS, resolve)
|
||||
custom = build_decision_maker(
|
||||
OracleDecisionMakerConfig(type="custom", path=f"{__name__}:CustomMaker"), MODELS, COSTS, resolve
|
||||
)
|
||||
assert isinstance(custom, DecisionMaker) and await custom.select(_ctx()) == "fast"
|
||||
with pytest.raises(TypeError):
|
||||
build_decision_maker(
|
||||
OracleDecisionMakerConfig(type="custom", path=f"{__name__}:NotAMaker"), MODELS, COSTS, resolve
|
||||
)
|
||||
|
||||
|
||||
def test_load_object_requires_module_colon_attribute():
|
||||
with pytest.raises(ValueError, match=r"package\.module:Attribute"):
|
||||
load_object("no_colon_here")
|
||||
assert load_object(f"{__name__}:CustomMaker") is CustomMaker
|
||||
|
||||
|
||||
def test_config_requires_the_fields_each_type_reads():
|
||||
with pytest.raises(ValueError, match=r"decision_maker\.model"):
|
||||
OracleDecisionMakerConfig(type="fixed")
|
||||
with pytest.raises(ValueError, match=r"decision_maker\.router"):
|
||||
OracleDecisionMakerConfig(type="pre_routing")
|
||||
with pytest.raises(ValueError, match=r"decision_maker\.path"):
|
||||
OracleDecisionMakerConfig(type="custom")
|
||||
200
tests/unit/router_strategy/oracle_router/test_hooks.py
Normal file
200
tests/unit/router_strategy/oracle_router/test_hooks.py
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
"""The post-call hook feeds spend and completion signals from LiteLLM's logging payload into the router."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.router_strategy.oracle_router.config import (
|
||||
CHOSEN_MODEL_METADATA_KEY,
|
||||
PROGRAM_ID_METADATA_KEY,
|
||||
RESPONSE_HEADER,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
|
||||
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
|
||||
from litellm.router_strategy.oracle_router.verifier import ReportedVerifier
|
||||
from litellm.types.router import OracleRouterConfig
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
|
||||
class _Maker:
|
||||
def __init__(self) -> None:
|
||||
self.updates = []
|
||||
|
||||
@property
|
||||
def models(self):
|
||||
return ("smart", "fast")
|
||||
|
||||
async def select(self, context):
|
||||
return "smart"
|
||||
|
||||
def update(self, context, model, score):
|
||||
self.updates.append((model, score))
|
||||
|
||||
def snapshot(self):
|
||||
return {}
|
||||
|
||||
|
||||
ROUTED = {"router_model_name": "oracle", "router_type": "oracle"} # what the Router stamps for this router
|
||||
|
||||
|
||||
def _kwargs(program_id="t1", cost=0.01, routing_decision=ROUTED, **metadata):
|
||||
stamped = {PROGRAM_ID_METADATA_KEY: program_id, **metadata}
|
||||
if routing_decision is not None:
|
||||
stamped["routing_decision"] = routing_decision
|
||||
return {
|
||||
"model": "openai/gpt-4o",
|
||||
"messages": [{"role": "user", "content": "task"}],
|
||||
"litellm_params": {"metadata": stamped},
|
||||
"response_cost": cost,
|
||||
}
|
||||
|
||||
|
||||
def _response(text="answer") -> ModelResponse:
|
||||
return ModelResponse(choices=[Choices(message=Message(content=text, role="assistant"))])
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bound():
|
||||
maker = _Maker()
|
||||
router = OracleRouter("oracle", OracleRouterConfig(available_models=["smart", "fast"]), maker, ReportedVerifier())
|
||||
asyncio.run(
|
||||
router.async_pre_routing_hook(
|
||||
model="oracle",
|
||||
request_kwargs={"metadata": {"program_id": "t1"}},
|
||||
messages=[{"role": "user", "content": "task"}],
|
||||
)
|
||||
)
|
||||
return router, maker, OracleRouterPostCallHook(router)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_success_events_accumulate_spend_and_the_latest_answer(bound):
|
||||
router, _, hook = bound
|
||||
await hook.async_log_success_event(_kwargs(cost=0.01), _response("first"), None, None)
|
||||
await hook.async_log_success_event(_kwargs(cost=0.02), _response("second"), None, None)
|
||||
binding = router.binding("t1")
|
||||
assert binding.requests == 2 and binding.cost == pytest.approx(0.03) and binding.last_response_text == "second"
|
||||
assert list(binding.last_messages) == [{"role": "user", "content": "task"}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_turn_that_opted_out_of_message_logging_marks_the_program_private_for_good(bound):
|
||||
router, _, hook = bound
|
||||
await hook.async_log_success_event(_kwargs(cost=0.01), _response("first"), None, None)
|
||||
assert router.binding("t1").message_logging_off is False
|
||||
await hook.async_log_success_event(
|
||||
_kwargs(cost=0.01, turn_off_message_logging=True), _response("second"), None, None
|
||||
)
|
||||
assert router.binding("t1").message_logging_off is True
|
||||
await hook.async_log_success_event(_kwargs(cost=0.01), _response("third"), None, None)
|
||||
assert router.binding("t1").message_logging_off is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_private_turn_that_failed_still_marks_the_program_private(bound):
|
||||
router, _, hook = bound
|
||||
await hook.async_log_failure_event(_kwargs(cost=0.5, turn_off_message_logging=True), None, None, None)
|
||||
assert router.binding("t1").message_logging_off is True and router.binding("t1").requests == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_turn_litellm_resolved_as_private_marks_the_program_private(bound):
|
||||
router, _, hook = bound
|
||||
resolved = {**_kwargs(cost=0.01), "standard_callback_dynamic_params": {"turn_off_message_logging": True}}
|
||||
await hook.async_log_success_event(resolved, _response("first"), None, None)
|
||||
assert router.binding("t1").message_logging_off is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_program_done_completes_the_program_with_the_reported_score(bound):
|
||||
router, maker, hook = bound
|
||||
await hook.async_log_success_event(
|
||||
_kwargs(cost=0.05, program_done=True, program_score="1"), _response(), None, None
|
||||
)
|
||||
assert router.binding("t1") is None
|
||||
await router.drain()
|
||||
assert maker.updates == [("smart", 1.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_non_finite_program_score_is_treated_as_no_score(bound):
|
||||
router, maker, hook = bound
|
||||
await hook.async_log_success_event(
|
||||
_kwargs(cost=0.05, program_done=True, program_score="nan"), _response(), None, None
|
||||
)
|
||||
assert router.binding("t1") is None
|
||||
await router.drain()
|
||||
assert maker.updates == [("smart", 0.0)] # the reported verifier's default, never NaN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_event_adds_no_spend_and_program_done_on_it_waits_for_a_response(bound):
|
||||
router, maker, hook = bound
|
||||
await hook.async_log_failure_event(_kwargs(cost=0.5), None, None, None)
|
||||
assert router.binding("t1").cost == 0.0
|
||||
await hook.async_log_failure_event(_kwargs(program_done=True), None, None, None)
|
||||
assert router.binding("t1") is not None and maker.updates == [] # no answer to verify yet: deferred
|
||||
await hook.async_log_success_event(_kwargs(cost=0.1), _response("late answer"), None, None)
|
||||
assert router.binding("t1") is None
|
||||
await router.drain()
|
||||
assert maker.updates == [("smart", 0.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_for_unknown_or_untagged_programs_are_ignored(bound):
|
||||
router, _, hook = bound
|
||||
await hook.async_log_success_event(
|
||||
{"litellm_params": {"metadata": {}}, "response_cost": 1.0}, _response(), None, None
|
||||
)
|
||||
await hook.async_log_success_event(_kwargs(program_id="other"), _response(), None, None)
|
||||
assert router.binding("t1").requests == 0 and router.binding("other") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_from_requests_this_router_did_not_route_are_ignored(bound):
|
||||
router, maker, hook = bound
|
||||
unrouted = _kwargs(cost=0.5, program_done=True, program_score="1", routing_decision=None)
|
||||
await hook.async_log_success_event(unrouted, _response(), None, None)
|
||||
elsewhere = _kwargs(
|
||||
cost=0.5, program_done=True, program_score="1", routing_decision={**ROUTED, "router_model_name": "x"}
|
||||
)
|
||||
await hook.async_log_success_event(elsewhere, _response(), None, None)
|
||||
assert router.binding("t1").requests == 0 and maker.updates == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_events_only_reach_the_program_of_the_key_that_sent_them(bound):
|
||||
router, maker, hook = bound # t1 was started without a key
|
||||
stranger = _kwargs(cost=0.5, program_done=True, program_score="1", user_api_key_hash="other-key")
|
||||
await hook.async_log_success_event(stranger, _response(), None, None)
|
||||
assert router.binding("t1").requests == 0 and maker.updates == []
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle",
|
||||
request_kwargs={"metadata": {"program_id": "t1", "user_api_key_hash": "other-key"}},
|
||||
messages=[{"role": "user", "content": "task"}],
|
||||
)
|
||||
await hook.async_log_success_event(stranger, _response(), None, None)
|
||||
assert router.binding("t1", "other-key") is None and router.binding("t1").requests == 0
|
||||
await router.drain()
|
||||
assert maker.updates == [("smart", 1.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_header_names_the_bound_model(bound):
|
||||
_, _, hook = bound
|
||||
headers = await hook.async_post_call_response_headers_hook(
|
||||
{"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart", "routing_decision": ROUTED}}, None, None
|
||||
)
|
||||
assert headers == {RESPONSE_HEADER: "smart"}
|
||||
assert (
|
||||
await hook.async_post_call_response_headers_hook({"metadata": {"routing_decision": ROUTED}}, None, None) is None
|
||||
)
|
||||
forged = {"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart"}} # a request this router did not route
|
||||
assert await hook.async_post_call_response_headers_hook(forged, None, None) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_header_reads_the_litellm_metadata_bucket_too(bound):
|
||||
_, _, hook = bound
|
||||
data = {"metadata": {}, "litellm_metadata": {CHOSEN_MODEL_METADATA_KEY: "fast", "routing_decision": ROUTED}}
|
||||
assert await hook.async_post_call_response_headers_hook(data, None, None) == {RESPONSE_HEADER: "fast"}
|
||||
445
tests/unit/router_strategy/oracle_router/test_oracle_router.py
Normal file
445
tests/unit/router_strategy/oracle_router/test_oracle_router.py
Normal file
|
|
@ -0,0 +1,445 @@
|
|||
"""OracleRouter: sticky program bindings, program id resolution, delayed feedback, eviction."""
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.core_helpers import budget_reservation_from_metadata
|
||||
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
|
||||
initialize_standard_callback_dynamic_params,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.config import (
|
||||
CHOSEN_MODEL_METADATA_KEY,
|
||||
PROGRAM_ID_HEADER,
|
||||
PROGRAM_ID_METADATA_KEY,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.decision import ProgramContext
|
||||
from litellm.router_strategy.oracle_router.oracle_router import (
|
||||
OracleRouter,
|
||||
VerificationBacklogFull,
|
||||
first_user_text,
|
||||
resolve_program_id,
|
||||
)
|
||||
from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, ReportedVerifier
|
||||
from litellm.types.router import OracleRouterConfig
|
||||
|
||||
MODELS = ("smart", "fast")
|
||||
|
||||
|
||||
class _RecordingMaker:
|
||||
"""Alternates models per bind and records every update in arrival order."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.selections = 0
|
||||
self.updates: list[tuple[str, str, float]] = []
|
||||
self.contexts: list[ProgramContext] = []
|
||||
|
||||
@property
|
||||
def models(self):
|
||||
return MODELS
|
||||
|
||||
async def select(self, context: ProgramContext) -> str:
|
||||
self.selections += 1
|
||||
self.contexts.append(context)
|
||||
return MODELS[(self.selections - 1) % 2]
|
||||
|
||||
def update(self, context: ProgramContext, model: str, score: float) -> None:
|
||||
self.updates.append((context.program_id, model, score))
|
||||
|
||||
def snapshot(self):
|
||||
return {"type": "recording"}
|
||||
|
||||
|
||||
class _SlowVerifier:
|
||||
def __init__(self, delays: dict[str, float]) -> None:
|
||||
self.delays = delays
|
||||
|
||||
async def verify(self, outcome: ProgramOutcome) -> float:
|
||||
await asyncio.sleep(self.delays.get(outcome.program_id, 0.0))
|
||||
return outcome.score if outcome.score is not None else 0.0
|
||||
|
||||
|
||||
def _router(verifier=None, maker=None, clock=None, **config) -> tuple[OracleRouter, _RecordingMaker]:
|
||||
maker = maker or _RecordingMaker()
|
||||
router = OracleRouter(
|
||||
router_name="oracle",
|
||||
config=OracleRouterConfig(available_models=list(MODELS), **config),
|
||||
decision_maker=maker,
|
||||
verifier=verifier or ReportedVerifier(),
|
||||
**({"clock": clock} if clock else {}),
|
||||
)
|
||||
return router, maker
|
||||
|
||||
|
||||
def _messages(text: str = "fix the failing test in src/parser.py"):
|
||||
return [{"role": "system", "content": "be terse"}, {"role": "user", "content": text}]
|
||||
|
||||
|
||||
def test_resolve_program_id_precedence_and_header():
|
||||
assert resolve_program_id({"metadata": {"program_id": "a", "litellm_session_id": "b"}}, "program_id") == "a"
|
||||
assert resolve_program_id({"metadata": {"litellm_session_id": "b"}}, "program_id") == "b"
|
||||
assert resolve_program_id({"metadata": {"session_id": 7}}, "program_id") == "7"
|
||||
assert resolve_program_id({"litellm_session_id": "lp"}, "program_id") == "lp"
|
||||
assert resolve_program_id({"headers": {PROGRAM_ID_HEADER.upper(): "h"}}, "program_id") == "h"
|
||||
assert resolve_program_id({"metadata": {"task": "t"}}, "task") == "t"
|
||||
assert resolve_program_id({"metadata": {"program_id": ""}}, "program_id") is None
|
||||
|
||||
|
||||
def test_first_user_text_reads_the_initial_request_including_text_parts():
|
||||
assert first_user_text(_messages("hello")) == "hello"
|
||||
parts = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": "a"}, {"type": "image_url"}, {"type": "text", "text": "b"}],
|
||||
}
|
||||
]
|
||||
assert first_user_text(parts) == "a\nb"
|
||||
assert first_user_text([{"role": "assistant", "content": "x"}]) == ""
|
||||
assert first_user_text(None) == ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_first_request_binds_and_later_requests_reuse_the_binding():
|
||||
router, maker = _router()
|
||||
first = {"metadata": {"program_id": "t1", "user_api_key_hash": "key-1"}}
|
||||
response = await router.async_pre_routing_hook(model="oracle", request_kwargs=first, messages=_messages())
|
||||
assert response is not None and response.model == "smart"
|
||||
assert (
|
||||
first["metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart" and first["metadata"][PROGRAM_ID_METADATA_KEY] == "t1"
|
||||
)
|
||||
assert response.routing_decision["router_type"] == "oracle" and response.routing_decision["routed_model"] == "smart"
|
||||
|
||||
later = {"metadata": {"program_id": "t1", "user_api_key_hash": "key-1"}}
|
||||
again = await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs=later, messages=_messages("now run the tests")
|
||||
)
|
||||
assert again is not None and again.model == "smart"
|
||||
assert maker.selections == 1
|
||||
assert router.binding("t1", "key-1").api_key_hash == "key-1"
|
||||
|
||||
other = await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "t2"}}, messages=_messages()
|
||||
)
|
||||
assert other.model == "fast" and router.programs_bound == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_concurrent_first_requests_of_one_program_share_a_single_binding():
|
||||
class _SlowMaker(_RecordingMaker):
|
||||
async def select(self, context: ProgramContext) -> str:
|
||||
await asyncio.sleep(0.01) # a decision maker that does I/O (pre_routing, custom)
|
||||
return await super().select(context)
|
||||
|
||||
router, maker = _router(maker=_SlowMaker())
|
||||
responses = await asyncio.gather(
|
||||
*(
|
||||
router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "t1"}}, messages=_messages()
|
||||
)
|
||||
for _ in range(3)
|
||||
)
|
||||
)
|
||||
assert {response.model for response in responses} == {"smart"}
|
||||
assert maker.selections == 1 and router.programs_bound == 1 and router.binding("t1").model == "smart"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_non_finite_verifier_score_is_a_failed_verification():
|
||||
class _NaN:
|
||||
async def verify(self, outcome: ProgramOutcome) -> float:
|
||||
return float("nan")
|
||||
|
||||
router, maker = _router(verifier=_NaN())
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
|
||||
)
|
||||
router.observe_request("t", cost=0.1, response_text="answer")
|
||||
result = await router.complete("t")
|
||||
assert result != result and router.verifications_failed == 1 and maker.updates == []
|
||||
assert router.binding("t") is None # the slot was released all the same
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_requests_without_a_program_id_route_statelessly_and_never_learn():
|
||||
router, maker = _router()
|
||||
kwargs = {"metadata": {}}
|
||||
response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages())
|
||||
assert response.model == "smart" and PROGRAM_ID_METADATA_KEY not in kwargs["metadata"]
|
||||
assert router.stateless_requests == 1 and router.binding("") is None
|
||||
assert router.complete("", score=1.0) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_releases_the_slot_now_and_learns_in_verification_order():
|
||||
router, maker = _router(verifier=_SlowVerifier({"t1": 0.05, "t2": 0.0}))
|
||||
for program_id in ("t1", "t2"):
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages()
|
||||
)
|
||||
router.observe_request("t1", cost=0.4, response_text="patch")
|
||||
router.observe_request("t1", cost=0.2, response_text="tests pass")
|
||||
router.observe_request("t2", cost=0.1, response_text="wrong")
|
||||
t1 = router.complete("t1", score=1.0)
|
||||
t2 = router.complete("t2", score=0.0)
|
||||
assert router.binding("t1") is None and router.binding("t2") is None
|
||||
assert router.pending_verifications == 2
|
||||
assert await t2 == 0.0
|
||||
assert await t1 == 1.0
|
||||
assert [u[0] for u in maker.updates] == ["t2", "t1"]
|
||||
assert maker.updates[1] == ("t1", "smart", 1.0)
|
||||
snapshot = await router.get_state_snapshot()
|
||||
assert snapshot["programs_completed"] == 2 and snapshot["recent_accuracy"] == {"smart": 1.0, "fast": 0.0}
|
||||
assert snapshot["recent_feedback"][-1]["cost"] == pytest.approx(0.6)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_full_verification_backlog_rejects_completion_and_keeps_the_program_bound():
|
||||
router, maker = _router(verifier=_SlowVerifier({"t1": 0.05}), max_pending_verifications=1)
|
||||
for program_id in ("t1", "t2"):
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages()
|
||||
)
|
||||
router.observe_request(program_id, cost=0.1, response_text="answer")
|
||||
first = router.complete("t1", score=1.0)
|
||||
with pytest.raises(VerificationBacklogFull, match="max_pending_verifications=1"):
|
||||
router.complete("t2", score=0.0)
|
||||
assert router.binding("t2") is not None and router.pending_verifications == 1
|
||||
assert router.verifications_rejected == 1 and router.programs_completed == 1
|
||||
assert await first == 1.0
|
||||
assert await router.complete("t2", score=0.0) == 0.0
|
||||
assert router.binding("t2") is None and [u[0] for u in maker.updates] == ["t1", "t2"]
|
||||
assert (await router.get_state_snapshot())["verifications_rejected"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_verification_that_outlives_its_timeout_fails_and_teaches_nothing():
|
||||
router, maker = _router(verifier=_SlowVerifier({"t": 5.0}), verification_timeout_seconds=0.01)
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
|
||||
)
|
||||
router.observe_request("t", cost=0.1, response_text="answer")
|
||||
task = router.complete("t", score=1.0)
|
||||
assert task is not None and math.isnan(await task)
|
||||
assert router.verifications_failed == 1 and maker.updates == []
|
||||
assert router.last_failure is not None and "verification: TimeoutError" in router.last_failure
|
||||
assert router.pending_verifications == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_program_s_message_logging_opt_out_is_kept_on_its_context():
|
||||
router, _ = _router()
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle",
|
||||
request_kwargs={"metadata": {"program_id": "private", "turn_off_message_logging": True}},
|
||||
messages=_messages(),
|
||||
)
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "public"}}, messages=_messages()
|
||||
)
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle",
|
||||
request_kwargs={
|
||||
"metadata": {"program_id": "by-header", "headers": {"litellm-enable-message-redaction": "true"}}
|
||||
},
|
||||
messages=_messages(),
|
||||
)
|
||||
assert router.binding("private").context.message_logging_off is True
|
||||
assert router.binding("by-header").context.message_logging_off is True
|
||||
assert router.binding("public").context.message_logging_off is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_decision_maker_sees_the_caller_s_identity_and_none_of_the_request_s_other_state():
|
||||
router, maker = _router()
|
||||
kwargs = {
|
||||
"metadata": {
|
||||
"program_id": "t",
|
||||
"user_api_key_hash": "hash-1",
|
||||
"user_api_key_team_id": "team-1",
|
||||
"user_api_key_budget_reservation": {"amount": 1.0},
|
||||
"user_api_key_auth": {"api_key": "hash-1", "budget_reservation": {"amount": 1.0}},
|
||||
"routing_decision": {"router_type": "oracle"},
|
||||
"custom": "value",
|
||||
},
|
||||
"litellm_metadata": {"user_api_key_user_id": "user-1", "_autorouter_baseline_route": "stamp"},
|
||||
}
|
||||
await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages())
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"litellm_metadata": {"user_api_key_hash": "hash-2"}}, messages=_messages()
|
||||
)
|
||||
bound, stateless = maker.contexts
|
||||
assert bound.caller == {
|
||||
"user_api_key_hash": "hash-1",
|
||||
"user_api_key_team_id": "team-1",
|
||||
"user_api_key_user_id": "user-1",
|
||||
"user_api_key_auth": {"api_key": "hash-1"},
|
||||
"internal_call_origin": "autorouter_classifier",
|
||||
}
|
||||
assert budget_reservation_from_metadata(dict(bound.caller)) is None
|
||||
assert stateless.caller == {"user_api_key_hash": "hash-2", "internal_call_origin": "autorouter_classifier"}
|
||||
assert router.context_for("", "task", {"metadata": {"routing_decision": {}}}).caller == {
|
||||
"internal_call_origin": "autorouter_classifier"
|
||||
}
|
||||
assert router.binding("t", "hash-1").context.caller == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_opt_out_on_a_later_turn_makes_the_whole_program_s_verification_private():
|
||||
seen: list[tuple[bool, bool | None]] = []
|
||||
|
||||
class _Peek:
|
||||
async def verify(self, outcome: ProgramOutcome) -> float:
|
||||
inherited = initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging")
|
||||
seen.append((outcome.message_logging_off, inherited))
|
||||
return 1.0
|
||||
|
||||
router, _ = _router(verifier=_Peek())
|
||||
for program_id in ("public", "late-private"):
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages()
|
||||
)
|
||||
router.observe_request("public", cost=0.1, response_text="a")
|
||||
router.observe_request("late-private", cost=0.1, response_text="a")
|
||||
router.mark_message_logging_off("late-private")
|
||||
router.observe_request("late-private", cost=0.1, response_text="c")
|
||||
assert router.binding("public").message_logging_off is False
|
||||
assert router.binding("late-private").message_logging_off is True
|
||||
assert await router.complete("public", score=1.0) == 1.0
|
||||
assert await router.complete("late-private", score=1.0) == 1.0
|
||||
assert seen == [(False, None), (True, True)]
|
||||
assert initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_responses_api_request_binds_its_program_on_the_input_text():
|
||||
router, _ = _router()
|
||||
response = await router.async_pre_routing_hook(
|
||||
model="oracle",
|
||||
request_kwargs={"metadata": {"program_id": "r"}},
|
||||
messages=None,
|
||||
input=[{"role": "user", "content": [{"type": "input_text", "text": "write a limerick"}]}],
|
||||
)
|
||||
assert response.messages is None and router.binding("r").context.prompt == "write a limerick"
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "s"}}, messages=None, input="plain text"
|
||||
)
|
||||
assert router.binding("s").context.prompt == "plain text"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_cost_override_and_payload_reach_the_verifier():
|
||||
seen: list[ProgramOutcome] = []
|
||||
|
||||
class _Capture:
|
||||
async def verify(self, outcome: ProgramOutcome) -> float:
|
||||
seen.append(outcome)
|
||||
return 1.0
|
||||
|
||||
router, maker = _router(verifier=_Capture())
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages("do x")
|
||||
)
|
||||
transcript = [{"role": "user", "content": "do x"}, {"role": "assistant", "content": "answer"}]
|
||||
router.observe_request("t", cost=0.3, response_text="answer", messages=transcript)
|
||||
await router.complete("t", cost=2.5, payload={"container": "c1"})
|
||||
assert seen[0].cost == 2.5 and seen[0].payload == {"container": "c1"} and seen[0].response_text == "answer"
|
||||
assert list(seen[0].messages) == transcript
|
||||
assert seen[0].prompt == "do x" and maker.updates == [("t", "smart", 1.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_verification_is_counted_and_does_not_update():
|
||||
class _Broken:
|
||||
async def verify(self, outcome):
|
||||
raise RuntimeError("sandbox down")
|
||||
|
||||
router, maker = _router(verifier=_Broken())
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
|
||||
)
|
||||
router.observe_request("t", cost=0.1, response_text="answer")
|
||||
result = await router.complete("t")
|
||||
assert result != result and router.verifications_failed == 1 and maker.updates == []
|
||||
snapshot = await router.get_state_snapshot()
|
||||
assert (
|
||||
snapshot["last_failure"] == "verification: RuntimeError: sandbox down"
|
||||
) # kept for the state endpoint, not logged
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_expired_programs_are_evicted_when_the_table_is_full():
|
||||
now = [1000.0]
|
||||
router, maker = _router(clock=lambda: now[0], program_ttl_seconds=10, max_programs=3)
|
||||
for program_id in ("a", "b", "c"):
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages()
|
||||
)
|
||||
now[0] += 4.0
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "d"}}, messages=_messages()
|
||||
)
|
||||
assert router.binding("a") is None and router.binding("b") is not None and router.binding("d") is not None
|
||||
assert router.programs_evicted == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_decision_is_stamped_in_the_proxy_internal_bucket_when_one_exists():
|
||||
router, _ = _router()
|
||||
kwargs = {"metadata": {"program_id": "t1"}, "litellm_metadata": {"user_api_key_hash": "k"}}
|
||||
response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages())
|
||||
assert response.model == "smart"
|
||||
assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart"
|
||||
assert CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"]
|
||||
assert router.binding("t1", "k").api_key_hash == "k"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_same_program_id_under_another_key_is_another_program():
|
||||
router, maker = _router()
|
||||
for key in ("key-a", "key-b"):
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle",
|
||||
request_kwargs={"metadata": {"program_id": "t1", "user_api_key_hash": key}},
|
||||
messages=_messages(),
|
||||
)
|
||||
assert router.programs_bound == 2 and maker.selections == 2
|
||||
assert router.binding("t1") is None # no key: not the same program either
|
||||
assert [binding.api_key_hash for binding in router.bindings_named("t1")] == ["key-a", "key-b"]
|
||||
router.observe_request("t1", cost=0.1, response_text="answer", owner="key-b")
|
||||
router.complete("t1", score=1.0, owner="key-b")
|
||||
assert router.binding("t1", "key-a") is not None and router.binding("t1", "key-b") is None
|
||||
await router.drain()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completion_before_any_observed_response_is_deferred_to_the_next_one():
|
||||
router, maker = _router()
|
||||
await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
|
||||
)
|
||||
assert router.complete("t", score=1.0, cost=2.0, payload={"run": "r1"}) is None # nothing answered yet
|
||||
binding = router.binding("t")
|
||||
assert binding is not None and binding.deferred is not None and router.programs_completed == 0
|
||||
assert maker.updates == []
|
||||
router.observe_request("t", cost=0.3, response_text="answer") # the response the feedback raced
|
||||
assert router.binding("t") is None and router.programs_completed == 1
|
||||
await router.drain()
|
||||
assert maker.updates == [("t", "smart", 1.0)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_routing_hook_replaces_caller_supplied_internal_keys():
|
||||
router, _ = _router()
|
||||
kwargs = {
|
||||
"metadata": {"program_id": "t1", PROGRAM_ID_METADATA_KEY: "victim", CHOSEN_MODEL_METADATA_KEY: "fast"},
|
||||
"litellm_metadata": {PROGRAM_ID_METADATA_KEY: "victim"},
|
||||
}
|
||||
response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages())
|
||||
assert PROGRAM_ID_METADATA_KEY not in kwargs["metadata"] and CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"]
|
||||
assert kwargs["litellm_metadata"][PROGRAM_ID_METADATA_KEY] == "t1"
|
||||
assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == response.model
|
||||
stateless = {"metadata": {PROGRAM_ID_METADATA_KEY: "victim", CHOSEN_MODEL_METADATA_KEY: "fast"}}
|
||||
await router.async_pre_routing_hook(model="oracle", request_kwargs=stateless, messages=_messages())
|
||||
assert PROGRAM_ID_METADATA_KEY not in stateless["metadata"]
|
||||
assert stateless["metadata"][CHOSEN_MODEL_METADATA_KEY] in ("smart", "fast")
|
||||
256
tests/unit/router_strategy/oracle_router/test_router_dispatch.py
Normal file
256
tests/unit/router_strategy/oracle_router/test_router_dispatch.py
Normal file
|
|
@ -0,0 +1,256 @@
|
|||
"""Router-level wiring of `auto_router/oracle_router`: registration, deferred init, dispatch, hooks, removal."""
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm import Router
|
||||
from litellm.router_strategy.oracle_router.config import PROGRAM_ID_METADATA_KEY
|
||||
from litellm.router_strategy.oracle_router.decision import PreRoutingDecisionMaker, ProgramContext
|
||||
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
|
||||
from litellm.types.router import Deployment, LiteLLM_Params, RequestType
|
||||
|
||||
|
||||
def _oracle(router: Router, name: str):
|
||||
return router.registry().plugins("oracle")[name][0].strategy
|
||||
|
||||
|
||||
def _model_list(decision_maker=None, verifier=None, extra=()):
|
||||
config = {"available_models": ["smart", "fast"]}
|
||||
if decision_maker:
|
||||
config["decision_maker"] = decision_maker
|
||||
if verifier:
|
||||
config["verifier"] = verifier
|
||||
return [
|
||||
{
|
||||
"model_name": "oracle",
|
||||
"litellm_params": {"model": "auto_router/oracle_router", "oracle_router_config": config},
|
||||
},
|
||||
{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}},
|
||||
{"model_name": "fast", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}},
|
||||
*extra,
|
||||
]
|
||||
|
||||
|
||||
def _oracle_hooks():
|
||||
return litellm.logging_callback_manager.get_custom_loggers_for_type(OracleRouterPostCallHook)
|
||||
|
||||
|
||||
class _CustomMaker:
|
||||
def __init__(self, models):
|
||||
self.models = tuple(models)
|
||||
|
||||
async def select(self, context):
|
||||
return self.models[-1]
|
||||
|
||||
def update(self, context, model, score):
|
||||
pass
|
||||
|
||||
def snapshot(self):
|
||||
return {"type": "custom"}
|
||||
|
||||
|
||||
def test_prefix_is_classified_as_oracle_and_not_as_a_semantic_router():
|
||||
router = Router(model_list=[])
|
||||
params = LiteLLM_Params(model="auto_router/oracle_router")
|
||||
assert router._is_oracle_router_deployment(litellm_params=params) is True
|
||||
assert router._is_auto_router_deployment(litellm_params=params) is False
|
||||
assert router._is_adaptive_router_deployment(litellm_params=params) is False
|
||||
assert router._is_oracle_router_deployment(litellm_params=LiteLLM_Params(model="openai/gpt-4o")) is False
|
||||
|
||||
|
||||
def test_init_registers_the_router_and_exactly_one_hook():
|
||||
router = Router(model_list=_model_list())
|
||||
assert "oracle" in router.registry().plugins("oracle")
|
||||
assert _oracle(router, "oracle").models == ("smart", "fast")
|
||||
hooks = [hook for hook in _oracle_hooks() if hook.oracle_router is _oracle(router, "oracle")]
|
||||
assert len(hooks) == 1
|
||||
|
||||
|
||||
def test_init_without_config_raises():
|
||||
with pytest.raises(ValueError, match="oracle_router_config is required"):
|
||||
Router(model_list=[{"model_name": "oracle", "litellm_params": {"model": "auto_router/oracle_router"}}])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_routing_hook_dispatches_to_the_oracle_router_and_binds_the_program():
|
||||
router = Router(model_list=_model_list(decision_maker={"type": "fixed", "model": "fast"}))
|
||||
kwargs = {"metadata": {"program_id": "task-1"}}
|
||||
response = await router.async_pre_routing_hook(
|
||||
model="oracle", request_kwargs=kwargs, messages=[{"role": "user", "content": "fix the bug"}]
|
||||
)
|
||||
assert response is not None and response.model == "fast"
|
||||
assert response.routing_decision["router_type"] == "oracle"
|
||||
assert kwargs["metadata"][PROGRAM_ID_METADATA_KEY] == "task-1"
|
||||
assert _oracle(router, "oracle").binding("task-1").model == "fast"
|
||||
|
||||
|
||||
def test_pre_routing_decision_maker_resolves_a_strategy_router_listed_after_it():
|
||||
quality = {
|
||||
"model_name": "tiered",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/quality_router",
|
||||
"quality_router_default_model": "fast",
|
||||
"quality_router_config": {"available_models": ["smart", "fast"]},
|
||||
},
|
||||
}
|
||||
model_list = _model_list(decision_maker={"type": "pre_routing", "router": "tiered"}, extra=(quality,))
|
||||
model_list[1]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 3}}
|
||||
model_list[2]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 1}}
|
||||
router = Router(model_list=model_list)
|
||||
assert isinstance(_oracle(router, "oracle").decision_maker, PreRoutingDecisionMaker)
|
||||
|
||||
|
||||
def test_init_reads_adaptive_router_preferences_as_bandit_priors():
|
||||
model_list = _model_list()
|
||||
model_list[1]["model_info"] = {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}
|
||||
router = Router(model_list=model_list)
|
||||
maker = _oracle(router, "oracle").decision_maker
|
||||
ctx = ProgramContext(program_id="p", prompt="x", request_type=RequestType.GENERAL)
|
||||
assert maker.estimate(ctx, "smart") > maker.estimate(ctx, "fast")
|
||||
|
||||
|
||||
def test_pre_routing_decision_maker_with_unknown_router_fails_init():
|
||||
with pytest.raises(ValueError, match="not a configured strategy router"):
|
||||
Router(model_list=_model_list(decision_maker={"type": "pre_routing", "router": "ghost"}))
|
||||
|
||||
|
||||
def test_hot_reload_keeps_the_existing_router_and_its_learned_state():
|
||||
router = Router(model_list=_model_list())
|
||||
before = _oracle(router, "oracle")
|
||||
router.set_model_list(_model_list())
|
||||
assert _oracle(router, "oracle") is before
|
||||
assert len([hook for hook in _oracle_hooks() if hook.oracle_router is before]) == 1
|
||||
|
||||
|
||||
def test_deleting_the_deployment_unregisters_the_router_and_its_hook():
|
||||
router = Router(model_list=_model_list())
|
||||
before = _oracle(router, "oracle")
|
||||
router.delete_deployment(id=router.get_model_ids(model_name="oracle")[0])
|
||||
assert "oracle" not in router.registry().plugins("oracle")
|
||||
assert all(hook.oracle_router is not before for hook in _oracle_hooks())
|
||||
|
||||
|
||||
def test_deployment_input_costs_prefer_litellm_params_over_model_info_and_skip_unknown_models():
|
||||
model_list = _model_list()
|
||||
model_list[1]["litellm_params"]["input_cost_per_token"] = 2e-6
|
||||
model_list[1]["model_info"] = {"input_cost_per_token": 9e-6}
|
||||
model_list[2]["model_info"] = {"input_cost_per_token": 1e-7}
|
||||
router = Router(model_list=model_list)
|
||||
assert router._deployment_input_costs(["smart", "fast", "ghost"]) == {"smart": 2e-6, "fast": 1e-7}
|
||||
|
||||
|
||||
def test_deployment_adaptive_prefs_read_model_info_where_declared():
|
||||
model_list = _model_list()
|
||||
model_list[1]["model_info"] = {"adaptive_router_preferences": {"quality_tier": 3, "strengths": ["code_generation"]}}
|
||||
router = Router(model_list=model_list)
|
||||
prefs = router._deployment_adaptive_prefs(["smart", "fast", "ghost"])
|
||||
assert set(prefs) == {"smart"} and prefs["smart"].quality_tier == 3
|
||||
assert prefs["smart"].strengths == [RequestType.CODE_GENERATION]
|
||||
|
||||
|
||||
def test_resolve_strategy_router_finds_registered_strategy_routers_only():
|
||||
quality = {
|
||||
"model_name": "tiered",
|
||||
"litellm_params": {
|
||||
"model": "auto_router/quality_router",
|
||||
"quality_router_default_model": "fast",
|
||||
"quality_router_config": {"available_models": ["smart", "fast"]},
|
||||
},
|
||||
}
|
||||
model_list = _model_list(extra=(quality,))
|
||||
model_list[1]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 3}}
|
||||
model_list[2]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 1}}
|
||||
router = Router(model_list=model_list)
|
||||
assert router._resolve_strategy_router("tiered") is router.registry().plugins("quality")["tiered"][0].strategy
|
||||
assert router._resolve_strategy_router("smart") is None and router._resolve_strategy_router("ghost") is None
|
||||
|
||||
|
||||
def test_init_oracle_router_deployment_registers_the_router_and_finalize_is_idempotent():
|
||||
router = Router(model_list=_model_list()[1:]) # the two plain deployments, no oracle router yet
|
||||
deployment = Deployment(
|
||||
model_name="late-oracle",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="auto_router/oracle_router", oracle_router_config={"available_models": ["smart", "fast"]}
|
||||
),
|
||||
)
|
||||
router.init_oracle_router_deployment(deployment=deployment)
|
||||
oracle = _oracle(router, "late-oracle")
|
||||
assert oracle.models == ("smart", "fast") and oracle.router_name == "late-oracle"
|
||||
router._sync_oracle_router_hooks()
|
||||
router._sync_oracle_router_hooks()
|
||||
assert len([hook for hook in _oracle_hooks() if hook.oracle_router is oracle]) == 1
|
||||
router._finalize_oracle_router_if_configured() # nothing new in model_list: the registry is untouched
|
||||
assert _oracle(router, "late-oracle") is oracle
|
||||
with pytest.raises(ValueError, match="oracle_router_config is required"):
|
||||
router.init_oracle_router_deployment(
|
||||
deployment=Deployment(model_name="bare", litellm_params=LiteLLM_Params(model="auto_router/oracle_router"))
|
||||
)
|
||||
|
||||
|
||||
def test_upserting_an_oracle_deployment_builds_its_router_once_and_rebuilds_it_when_its_config_changes():
|
||||
router = Router(model_list=_model_list()[1:]) # the two plain deployments, no oracle router yet
|
||||
deployment = Deployment(
|
||||
model_name="late-oracle",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="auto_router/oracle_router", oracle_router_config={"available_models": ["smart", "fast"]}
|
||||
),
|
||||
model_info={"id": "oracle-1"},
|
||||
)
|
||||
router.upsert_deployment(deployment=deployment) # a new id: nothing was on the router before
|
||||
oracle = _oracle(router, "late-oracle")
|
||||
assert oracle.models == ("smart", "fast")
|
||||
router.upsert_deployment(deployment=deployment) # unchanged: the router and its learned state survive
|
||||
assert _oracle(router, "late-oracle") is oracle
|
||||
assert len([hook for hook in _oracle_hooks() if hook.oracle_router is oracle]) == 1
|
||||
router.upsert_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="late-oracle",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="auto_router/oracle_router", oracle_router_config={"available_models": ["fast"]}
|
||||
),
|
||||
model_info={"id": "oracle-1"},
|
||||
)
|
||||
)
|
||||
rebuilt = _oracle(router, "late-oracle")
|
||||
assert rebuilt is not oracle and rebuilt.models == ("fast",)
|
||||
assert [hook.oracle_router for hook in _oracle_hooks() if hook.oracle_router in (oracle, rebuilt)] == [rebuilt]
|
||||
|
||||
|
||||
def test_custom_plugins_load_from_the_config_file_but_never_for_models_written_through_the_api(monkeypatch):
|
||||
import subprocess
|
||||
|
||||
popen_calls = []
|
||||
monkeypatch.setattr(subprocess, "Popen", lambda *args, **kwargs: popen_calls.append(args))
|
||||
from_config = Router(model_list=_model_list(decision_maker={"type": "custom", "path": f"{__name__}:_CustomMaker"}))
|
||||
assert _oracle(from_config, "oracle").decision_maker.models == ("smart", "fast")
|
||||
for config_key in ("decision_maker", "verifier"):
|
||||
model_list = _model_list(**{config_key: {"type": "custom", "path": "subprocess:Popen"}})
|
||||
model_list[0]["model_info"] = {"db_model": True} # what the proxy sets for /model/new and /model/update
|
||||
model_list[0]["litellm_params"]["oracle_router_config"]["available_models"] = ["/bin/sh", "-c", "id"]
|
||||
with pytest.raises(ValueError, match="config file"):
|
||||
Router(model_list=model_list)
|
||||
router = Router(model_list=_model_list()[1:])
|
||||
with pytest.raises(ValueError, match="config file"):
|
||||
router.upsert_deployment(
|
||||
deployment=Deployment(
|
||||
model_name="api-oracle",
|
||||
litellm_params=LiteLLM_Params(
|
||||
model="auto_router/oracle_router",
|
||||
oracle_router_config={
|
||||
"available_models": ["smart", "fast"],
|
||||
"verifier": {"type": "custom", "path": "subprocess:Popen"},
|
||||
},
|
||||
),
|
||||
model_info={"id": "api-oracle-1", "db_model": True},
|
||||
)
|
||||
)
|
||||
assert popen_calls == [] and "api-oracle" not in router.registry().plugins("oracle")
|
||||
|
||||
|
||||
def test_strategy_router_dependencies_cover_available_models():
|
||||
from litellm.router_strategy.auto_router.auto_router_model_naming import strategy_router_dependencies
|
||||
|
||||
deps = strategy_router_dependencies(
|
||||
{"model": "auto_router/oracle_router", "oracle_router_config": {"available_models": ["smart", "fast"]}}
|
||||
)
|
||||
assert {(dep.model_name, dep.role) for dep in deps} == {("smart", "tier"), ("fast", "tier")}
|
||||
311
tests/unit/router_strategy/oracle_router/test_verifier.py
Normal file
311
tests/unit/router_strategy/oracle_router/test_verifier.py
Normal file
|
|
@ -0,0 +1,311 @@
|
|||
"""Verifiers: LiteLLM guardrails and reported scores become the number the decision maker learns from."""
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.constants import REDACTED_BY_LITELLM
|
||||
from litellm.exceptions import GuardrailRaisedException
|
||||
from litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge import LLMAsAJudgeGuardrail
|
||||
from litellm.router_strategy.oracle_router.verifier import (
|
||||
GuardrailVerifier,
|
||||
ProgramOutcome,
|
||||
ReportedVerifier,
|
||||
Verifier,
|
||||
build_verifier,
|
||||
clamp_score,
|
||||
recorded_status,
|
||||
response_text,
|
||||
)
|
||||
from litellm.types.router import OracleVerifierConfig
|
||||
from litellm.types.utils import Choices, Message, ModelResponse
|
||||
|
||||
|
||||
def _outcome(score=None, cost=0.0, answer="done", messages=()) -> ProgramOutcome:
|
||||
return ProgramOutcome(
|
||||
program_id="p", model="smart", prompt="task", response_text=answer, cost=cost, messages=messages, score=score
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reported_verifier_uses_the_reported_score_or_its_default():
|
||||
assert await ReportedVerifier().verify(_outcome(score=0.8)) == 0.8
|
||||
assert await ReportedVerifier().verify(_outcome(score=4.0)) == 1.0
|
||||
assert await ReportedVerifier(default=0.25).verify(_outcome()) == 0.25
|
||||
|
||||
|
||||
class _Guardrail:
|
||||
"""A guardrail that blocks the way LiteLLM's do: by raising, with no status record."""
|
||||
|
||||
def __init__(self, blocks: bool) -> None:
|
||||
self.blocks = blocks
|
||||
self.seen: list[dict] = []
|
||||
|
||||
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
|
||||
self.seen.append({"inputs": inputs, "input_type": input_type, "request_data": request_data})
|
||||
if self.blocks:
|
||||
raise GuardrailRaisedException(guardrail_name="strict", message="blocked", blocked_content=True)
|
||||
return inputs
|
||||
|
||||
|
||||
class _RecordingGuardrail:
|
||||
"""A guardrail that reports through LiteLLM's status record, like llm_as_a_judge, and may also raise."""
|
||||
|
||||
def __init__(self, status: str, error: Exception | None = None, name: str = "judge") -> None:
|
||||
self.status, self.error, self.name = status, error, name
|
||||
|
||||
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
|
||||
self.seen = dict(request_data["metadata"]) # what the guardrail was handed, before it records anything
|
||||
records = request_data["metadata"].setdefault("standard_logging_guardrail_information", [])
|
||||
records.append({"guardrail_name": self.name, "guardrail_status": self.status}) # appended, like the real ones
|
||||
if self.error is not None:
|
||||
raise self.error
|
||||
return inputs
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_payload_cannot_pose_as_the_proxy_to_the_guardrail():
|
||||
judge = _RecordingGuardrail("success")
|
||||
outcome = ProgramOutcome(
|
||||
program_id="p",
|
||||
model="smart",
|
||||
prompt="task",
|
||||
response_text="answer",
|
||||
cost=0.1,
|
||||
payload={
|
||||
"container": "c1",
|
||||
"user_api_key_team_id": "someone-else",
|
||||
"user_api_key": "sk-forged",
|
||||
"internal_call_origin": "shadow_eval_judge",
|
||||
"headers": {"litellm-disable-message-redaction": "true"},
|
||||
"routing_decision": {"router_type": "oracle"},
|
||||
},
|
||||
)
|
||||
assert await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(outcome) == 1.0
|
||||
assert judge.seen == {"container": "c1", "program_id": "p"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_verifier_refuses_a_final_answer_litellm_redacted_instead_of_judging_the_placeholder():
|
||||
judge = _Guardrail(blocks=False)
|
||||
outcome = ProgramOutcome(program_id="p", model="smart", prompt="task", response_text=REDACTED_BY_LITELLM, cost=0.1)
|
||||
with pytest.raises(RuntimeError, match="redacted before verification"):
|
||||
await GuardrailVerifier("ok", lookup={"ok": judge}.get).verify(outcome)
|
||||
stale_answer_behind_a_redacted_turn = ProgramOutcome(
|
||||
program_id="p",
|
||||
model="smart",
|
||||
prompt="task",
|
||||
response_text="an earlier public answer",
|
||||
cost=0.1,
|
||||
messages=[{"role": "user", "content": REDACTED_BY_LITELLM}],
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="redacted before verification"):
|
||||
await GuardrailVerifier("ok", lookup={"ok": judge}.get).verify(stale_answer_behind_a_redacted_turn)
|
||||
assert judge.seen == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_verifier_hands_the_judge_the_transcript_and_the_feedback_payload():
|
||||
passing, blocking = _Guardrail(blocks=False), _Guardrail(blocks=True)
|
||||
lookup = {"ok": passing, "strict": blocking}.get
|
||||
transcript = ({"role": "user", "content": "task"}, {"role": "assistant", "content": "final answer"})
|
||||
outcome = ProgramOutcome(
|
||||
program_id="p",
|
||||
model="smart",
|
||||
prompt="task",
|
||||
response_text="final answer",
|
||||
cost=0.0,
|
||||
messages=transcript,
|
||||
payload={"reference": "42"},
|
||||
)
|
||||
assert await GuardrailVerifier("ok", lookup=lookup).verify(outcome) == 1.0
|
||||
assert passing.seen == [
|
||||
{
|
||||
"inputs": {"texts": ["final answer"], "model": "smart", "structured_messages": list(transcript)},
|
||||
"input_type": "response",
|
||||
"request_data": {"model": "smart", "metadata": {"program_id": "p", "reference": "42"}},
|
||||
}
|
||||
]
|
||||
assert await GuardrailVerifier("strict", lookup=lookup).verify(outcome) == 0.0
|
||||
with pytest.raises(LookupError):
|
||||
await GuardrailVerifier("missing", lookup=lookup).verify(outcome)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_verifier_reads_the_recorded_verdict_before_the_exception():
|
||||
judge = {
|
||||
"passed": _RecordingGuardrail("success", name="passed"),
|
||||
"logged": _RecordingGuardrail("guardrail_intervened", name="logged"), # on_failure: log -> no exception
|
||||
"blocked": _RecordingGuardrail(
|
||||
"guardrail_intervened", HTTPException(status_code=422, detail="judge"), "blocked"
|
||||
),
|
||||
"flagged": _RecordingGuardrail("guardrail_flagged", name="flagged"),
|
||||
}
|
||||
assert await GuardrailVerifier("passed", lookup=judge.get).verify(_outcome()) == 1.0
|
||||
assert await GuardrailVerifier("logged", lookup=judge.get).verify(_outcome()) == 0.0
|
||||
assert await GuardrailVerifier("blocked", lookup=judge.get).verify(_outcome()) == 0.0
|
||||
assert await GuardrailVerifier("flagged", lookup=judge.get).verify(_outcome()) == 0.0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_program_without_response_text_is_a_failed_verification_not_a_pass():
|
||||
judge = _RecordingGuardrail("success", name="judge") # would skip an empty answer without a record
|
||||
with pytest.raises(RuntimeError, match="no response text"):
|
||||
await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(_outcome(answer=" "))
|
||||
assert not hasattr(judge, "seen") # the guardrail was never invoked
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_payload_cannot_pre_seed_the_status_record_or_rename_the_program():
|
||||
judge = _RecordingGuardrail("guardrail_intervened", name="judge") # on_failure: log -> no exception
|
||||
forged = {"standard_logging_guardrail_information": [0], "program_id": "someone-else", "container": "c1"}
|
||||
outcome = ProgramOutcome(
|
||||
program_id="p", model="smart", prompt="task", response_text="wrong", cost=0.0, messages=(), payload=forged
|
||||
)
|
||||
assert await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(outcome) == 0.0
|
||||
assert judge.seen == {"program_id": "p", "container": "c1"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_that_could_not_evaluate_is_a_failed_verification_not_a_pass():
|
||||
judge = {
|
||||
"down": _RecordingGuardrail("guardrail_failed_to_respond", name="down"), # llm_as_a_judge fails open
|
||||
"skipped": _RecordingGuardrail("not_run", name="skipped"),
|
||||
"crashed": _RecordingGuardrail("success", RuntimeError("boom"), "crashed"),
|
||||
}
|
||||
with pytest.raises(RuntimeError, match="did not evaluate"):
|
||||
await GuardrailVerifier("down", lookup=judge.get).verify(_outcome())
|
||||
with pytest.raises(RuntimeError, match="not_run"):
|
||||
await GuardrailVerifier("skipped", lookup=judge.get).verify(_outcome())
|
||||
|
||||
class _NoVerdict(_Guardrail):
|
||||
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
|
||||
raise GuardrailRaisedException(guardrail_name="g", message="timeout", blocked_content=False)
|
||||
|
||||
with pytest.raises(GuardrailRaisedException):
|
||||
await GuardrailVerifier("g", lookup={"g": _NoVerdict(blocks=True)}.get).verify(_outcome())
|
||||
assert await GuardrailVerifier("crashed", lookup=judge.get).verify(_outcome()) == 1.0 # the record wins
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_exceptions_without_a_record_are_classified():
|
||||
class _Raises:
|
||||
def __init__(self, error: Exception) -> None:
|
||||
self.error = error
|
||||
|
||||
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
|
||||
raise self.error
|
||||
|
||||
lookup = {
|
||||
"http": _Raises(HTTPException(status_code=400, detail="Violated guardrail policy")),
|
||||
"infra": _Raises(ConnectionError("provider down")),
|
||||
}.get
|
||||
assert await GuardrailVerifier("http", lookup=lookup).verify(_outcome()) == 0.0
|
||||
with pytest.raises(ConnectionError):
|
||||
await GuardrailVerifier("infra", lookup=lookup).verify(_outcome())
|
||||
|
||||
|
||||
def _judge(monkeypatch, on_failure: str, verdict) -> LLMAsAJudgeGuardrail:
|
||||
"""The real llm_as_a_judge guardrail with only its judge-model call stubbed."""
|
||||
guardrail = LLMAsAJudgeGuardrail(
|
||||
guardrail_name="answer-judge",
|
||||
judge_model="judge",
|
||||
criteria=[{"name": "task_completed", "weight": 100, "description": "the task was completed"}],
|
||||
overall_threshold=50,
|
||||
on_failure=on_failure,
|
||||
)
|
||||
|
||||
async def run_judge(messages, text_under_review, input_type="response"):
|
||||
if isinstance(verdict, Exception):
|
||||
raise verdict
|
||||
return verdict
|
||||
|
||||
monkeypatch.setattr(guardrail, "_run_judge", run_judge)
|
||||
return guardrail
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("on_failure", "overall_score", "expected"),
|
||||
[
|
||||
("block", 90, 1.0), # passes the threshold: recorded as success
|
||||
("block", 10, 0.0), # fails: the judge raises 422 and records guardrail_intervened
|
||||
("log", 10, 0.0), # fails but only logs: no exception, still recorded as guardrail_intervened
|
||||
],
|
||||
)
|
||||
async def test_real_llm_as_a_judge_verdicts_become_scores(monkeypatch, on_failure, overall_score, expected):
|
||||
guardrail = _judge(monkeypatch, on_failure, {"overall_score": overall_score, "verdicts": []})
|
||||
verifier = GuardrailVerifier("answer-judge", lookup={"answer-judge": guardrail}.get)
|
||||
assert await verifier.verify(_outcome(answer="final answer")) == expected
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_real_llm_as_a_judge_that_fails_open_is_not_a_pass(monkeypatch):
|
||||
guardrail = _judge(monkeypatch, "block", RuntimeError("judge model unreachable")) # the guardrail returns normally
|
||||
verifier = GuardrailVerifier("answer-judge", lookup={"answer-judge": guardrail}.get)
|
||||
with pytest.raises(RuntimeError, match="did not evaluate"):
|
||||
await verifier.verify(_outcome(answer="final answer"))
|
||||
|
||||
|
||||
def test_recorded_status_reads_both_buckets_and_takes_the_latest():
|
||||
data = {
|
||||
"metadata": {
|
||||
"standard_logging_guardrail_information": [{"guardrail_name": "a", "guardrail_status": "success"}]
|
||||
},
|
||||
"litellm_metadata": {
|
||||
"standard_logging_guardrail_information": {"guardrail_name": "a", "guardrail_status": "guardrail_flagged"}
|
||||
},
|
||||
}
|
||||
assert recorded_status(data, "a") == "success" # a single dict, not a list, is not a valid record set
|
||||
data["litellm_metadata"]["standard_logging_guardrail_information"] = [
|
||||
{"guardrail_name": "a", "guardrail_status": "guardrail_intervened"}
|
||||
]
|
||||
assert recorded_status(data, "a") == "guardrail_intervened"
|
||||
assert recorded_status(data, "other") is None
|
||||
assert recorded_status({"metadata": "junk"}, "a") is None
|
||||
|
||||
|
||||
def test_clamp_score_rejects_non_finite_scores():
|
||||
assert clamp_score(1.7) == 1.0 and clamp_score(-2) == 0.0
|
||||
for bad in (float("nan"), float("inf")):
|
||||
with pytest.raises(ValueError, match="finite"):
|
||||
clamp_score(bad)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reported_verifier_rejects_a_nan_score():
|
||||
with pytest.raises(ValueError, match="finite"):
|
||||
await ReportedVerifier().verify(_outcome(score=float("nan")))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_guardrail_verifier_without_a_transcript_sends_the_initial_request():
|
||||
passing = _Guardrail(blocks=False)
|
||||
await GuardrailVerifier("ok", lookup={"ok": passing}.get).verify(_outcome())
|
||||
assert passing.seen[0]["inputs"]["structured_messages"] == [{"role": "user", "content": "task"}]
|
||||
|
||||
|
||||
def test_response_text_reads_chat_completions_only():
|
||||
assert response_text(ModelResponse(choices=[Choices(message=Message(content="hi", role="assistant"))])) == "hi"
|
||||
assert response_text({"choices": []}) == ""
|
||||
assert response_text(None) == ""
|
||||
|
||||
|
||||
class CustomVerifier:
|
||||
async def verify(self, outcome: ProgramOutcome) -> float:
|
||||
return 0.5
|
||||
|
||||
|
||||
class NotAVerifier:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_verifier_by_type():
|
||||
assert isinstance(build_verifier(OracleVerifierConfig(type="reported")), ReportedVerifier)
|
||||
assert isinstance(build_verifier(OracleVerifierConfig(type="guardrail", guardrail="g")), GuardrailVerifier)
|
||||
custom = build_verifier(OracleVerifierConfig(type="custom", path=f"{__name__}:CustomVerifier"))
|
||||
assert isinstance(custom, Verifier) and await custom.verify(_outcome()) == 0.5
|
||||
with pytest.raises(TypeError):
|
||||
build_verifier(OracleVerifierConfig(type="custom", path=f"{__name__}:NotAVerifier"))
|
||||
with pytest.raises(ValueError, match=r"verifier\.guardrail"):
|
||||
OracleVerifierConfig(type="guardrail")
|
||||
154
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
154
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -11713,6 +11713,51 @@ export interface paths {
|
|||
patch: operations["openrouter_proxy_route_openrouter__endpoint__patch"];
|
||||
trace?: never;
|
||||
};
|
||||
"/oracle_router/feedback": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
get?: never;
|
||||
put?: never;
|
||||
/**
|
||||
* Submit Oracle Router Feedback
|
||||
* @description Complete a program: its model slot is released now, its verifier runs in the background.
|
||||
*
|
||||
* Programs are private to the key that started them; an admin may complete any key's program. 404 when
|
||||
* the caller holds no program with that id, which also covers programs completed earlier. A program none
|
||||
* of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed
|
||||
* response completes it with this feedback.
|
||||
*/
|
||||
post: operations["submit_oracle_router_feedback_oracle_router_feedback_post"];
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/oracle_router/state": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
/**
|
||||
* Get Oracle Router State
|
||||
* @description Decision-maker state, active bindings and recent verified feedback for every ORACLE router. Admin only.
|
||||
*/
|
||||
get: operations["get_oracle_router_state_oracle_router_state_get"];
|
||||
put?: never;
|
||||
post?: never;
|
||||
delete?: never;
|
||||
options?: never;
|
||||
head?: never;
|
||||
patch?: never;
|
||||
trace?: never;
|
||||
};
|
||||
"/organization/daily/activity": {
|
||||
parameters: {
|
||||
query?: never;
|
||||
|
|
@ -36669,6 +36714,10 @@ export interface components {
|
|||
openai_identity_token_file?: string | null;
|
||||
/** Openai Service Account Id */
|
||||
openai_service_account_id?: string | null;
|
||||
/** Oracle Router Config */
|
||||
oracle_router_config?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Organization */
|
||||
organization?: string | null;
|
||||
/** Otpm */
|
||||
|
|
@ -41233,6 +41282,52 @@ export interface components {
|
|||
} & {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/**
|
||||
* OracleRouterFeedbackRequest
|
||||
* @description Report that a program finished. ``score`` is optional when the router's verifier grades it.
|
||||
*/
|
||||
OracleRouterFeedbackRequest: {
|
||||
/**
|
||||
* Cost
|
||||
* @description Overrides the spend LiteLLM accumulated for the program
|
||||
*/
|
||||
cost?: number | null;
|
||||
/**
|
||||
* Payload
|
||||
* @description Passed to the verifier unchanged
|
||||
*/
|
||||
payload?: {
|
||||
[key: string]: unknown;
|
||||
};
|
||||
/** Program Id */
|
||||
program_id: string;
|
||||
/** Score */
|
||||
score?: number | null;
|
||||
};
|
||||
/** OracleRouterFeedbackResponse */
|
||||
OracleRouterFeedbackResponse: {
|
||||
/**
|
||||
* Deferred
|
||||
* @description No response of the program has been observed yet; its next one completes it with this feedback
|
||||
* @default false
|
||||
*/
|
||||
deferred: boolean;
|
||||
/** Model */
|
||||
model: string;
|
||||
/** Pending Verifications */
|
||||
pending_verifications: number;
|
||||
/** Program Id */
|
||||
program_id: string;
|
||||
/** Router Name */
|
||||
router_name: string;
|
||||
};
|
||||
/** OracleRouterStateResponse */
|
||||
OracleRouterStateResponse: {
|
||||
/** Routers */
|
||||
routers: {
|
||||
[key: string]: unknown;
|
||||
}[];
|
||||
};
|
||||
/** OrgMember */
|
||||
OrgMember: {
|
||||
/**
|
||||
|
|
@ -47577,7 +47672,7 @@ export interface components {
|
|||
* Router Type
|
||||
* @enum {string}
|
||||
*/
|
||||
router_type?: "complexity" | "adaptive" | "quality" | "semantic";
|
||||
router_type?: "complexity" | "adaptive" | "quality" | "semantic" | "oracle";
|
||||
/** Savings Baseline Deployment Id */
|
||||
savings_baseline_deployment_id?: string;
|
||||
/** Savings Baseline Model */
|
||||
|
|
@ -52733,6 +52828,10 @@ export interface components {
|
|||
openai_identity_token_file?: string | null;
|
||||
/** Openai Service Account Id */
|
||||
openai_service_account_id?: string | null;
|
||||
/** Oracle Router Config */
|
||||
oracle_router_config?: {
|
||||
[key: string]: unknown;
|
||||
} | null;
|
||||
/** Organization */
|
||||
organization?: string | null;
|
||||
/** Otpm */
|
||||
|
|
@ -69960,6 +70059,59 @@ export interface operations {
|
|||
};
|
||||
};
|
||||
};
|
||||
submit_oracle_router_feedback_oracle_router_feedback_post: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody: {
|
||||
content: {
|
||||
"application/json": components["schemas"]["OracleRouterFeedbackRequest"];
|
||||
};
|
||||
};
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["OracleRouterFeedbackResponse"];
|
||||
};
|
||||
};
|
||||
/** @description Validation Error */
|
||||
422: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["HTTPValidationError"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_oracle_router_state_oracle_router_state_get: {
|
||||
parameters: {
|
||||
query?: never;
|
||||
header?: never;
|
||||
path?: never;
|
||||
cookie?: never;
|
||||
};
|
||||
requestBody?: never;
|
||||
responses: {
|
||||
/** @description Successful Response */
|
||||
200: {
|
||||
headers: {
|
||||
[name: string]: unknown;
|
||||
};
|
||||
content: {
|
||||
"application/json": components["schemas"]["OracleRouterStateResponse"];
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
get_organization_daily_activity_organization_daily_activity_get: {
|
||||
parameters: {
|
||||
query?: {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue