This commit is contained in:
Ritik Raj 2026-11-05 00:40:32 +00:00 • committed by GitHub
commit b50a7ed799
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
27 changed files with 3529 additions and 14 deletions

View file

@ -123,6 +123,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
# Realtime / streaming
"/v1/realtime",
"/realtime",
# ORACLE router feedback and state: the program bindings live in the serving process
"/oracle_router/",
"/v1/live/sessions",
"/live/sessions",
# Health & ops

View file

@ -0,0 +1,51 @@
# ORACLE router: one model per agentic program, learned from verified outcomes.
#
# Clients call `model: oracle-router` and tag every request of a task with the same program id:
# POST /v1/chat/completions { "model": "oracle-router", "metadata": { "program_id": "task-17" }, ... }
# The first request binds the program to a model; later requests reuse it. On the last request add
# "metadata": { "program_id": "task-17", "program_done": true, "program_score": 1.0 }
# or POST /oracle_router/feedback { "program_id": "task-17", "score": 1.0 } once the task is graded.
#
# Required env vars: OPENAI_API_KEY, LITELLM_MASTER_KEY.
model_list:
- model_name: oracle-router
litellm_params:
model: auto_router/oracle_router
oracle_router_config:
available_models: ["smart", "fast"] # strongest first
decision_maker:
type: thompson # fixed | pre_routing | custom
weights: { quality: 0.7, cost: 0.3 } # the adaptive router's quality/cost trade-off
verifier:
type: guardrail # reported | custom
guardrail: answer-judge
# Let LiteLLM's complexity router propose the model and have ORACLE bind, verify and report it:
# decision_maker: { type: pre_routing, router: tiered-router }
- model_name: smart
litellm_params:
model: openai/gpt-4o
api_key: os.environ/OPENAI_API_KEY
- model_name: fast
litellm_params:
model: openai/gpt-4o-mini
api_key: os.environ/OPENAI_API_KEY
guardrails:
- guardrail_name: answer-judge
litellm_params:
guardrail: llm_as_a_judge
mode: post_call
default_on: false
judge_model: smart
overall_threshold: 50
criteria:
- name: task_completed
weight: 100
description: The assistant completed the user's task correctly, judged from the whole conversation.
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY

View file

@ -3970,7 +3970,7 @@ async def clear_cache() -> ReconcileOutcome:
config_models: Final = []
db_model_ids: Final = []
db_router_names: Final = set()
db_router_names: Final[set[str]] = set() # mutable-ok: filled in the classification pass below
for model in current_models:
model_info = model.get("model_info", {})
@ -3979,7 +3979,7 @@ async def clear_cache() -> ReconcileOutcome:
# Auto-router deployments (and only those) are wiped here, in the
# same pass, so the reload rebuilds them -- see the comment below.
model_name = model.get("model_name")
if model_name is not None and str(model.get("litellm_params", {}).get("model", "")).startswith(
if isinstance(model_name, str) and str(model.get("litellm_params", {}).get("model", "")).startswith(
"auto_router/"
):
db_router_names.add(model_name)
@ -4028,6 +4028,7 @@ async def clear_cache() -> ReconcileOutcome:
llm_router.registry().plugins("complexity").pop(model_name, None)
llm_router.registry().plugins("adaptive").pop(model_name, None)
llm_router.registry().plugins("quality").pop(model_name, None)
llm_router.registry().plugins("oracle").pop(model_name, None)
# Reload only DB models. _add_deployment_locked, not add_deployment: this
# coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not

View file

@ -0,0 +1,117 @@
"""
ORACLE ROUTER MANAGEMENT ENDPOINTS
POST /oracle_router/feedback - Report a finished program so its verifier runs and the router learns
GET /oracle_router/state - Live decision-maker state, bindings and recent feedback per router
"""
from itertools import chain
from typing import TYPE_CHECKING, Annotated, Final
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.router_strategy.oracle_router.oracle_router import VerificationBacklogFull
from litellm.types.management_endpoints.oracle_router_endpoints import (
OracleRouterFeedbackRequest,
OracleRouterFeedbackResponse,
OracleRouterStateResponse,
)
if TYPE_CHECKING:
from fastapi import APIRouter, Depends, HTTPException
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, ProgramBinding
else:
try:
from fastapi import APIRouter, Depends, HTTPException
except ImportError:
pass
router: Final = APIRouter()
def _is_admin(user_api_key_dict: UserAPIKeyAuth) -> bool:
return user_api_key_dict.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY)
def _oracle_routers() -> tuple["OracleRouter", ...]:
from litellm.proxy.proxy_server import llm_router
registry: Final = llm_router.registry().plugins("oracle") if llm_router is not None else {}
tagged_entries: Final = tuple(chain.from_iterable(registry.values()))
return tuple(tagged.strategy for tagged in tagged_entries)
def _find_program(
routers: tuple["OracleRouter", ...], program_id: str, user_api_key_dict: UserAPIKeyAuth
) -> tuple["OracleRouter", "ProgramBinding"] | None:
"""The caller's program with this id. An admin may also reach another key's program of that id."""
for candidate in routers:
owned = candidate.binding(program_id, user_api_key_dict.api_key)
if owned is not None:
return candidate, owned
if not _is_admin(user_api_key_dict):
return None
for candidate in routers:
for any_owner in candidate.bindings_named(program_id):
return candidate, any_owner
return None
@router.post(
"/oracle_router/feedback",
tags=["oracle_router"],
dependencies=[Depends(user_api_key_auth)],
response_model=OracleRouterFeedbackResponse,
)
async def submit_oracle_router_feedback(
data: OracleRouterFeedbackRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> OracleRouterFeedbackResponse:
"""Complete a program: its model slot is released now, its verifier runs in the background.
Programs are private to the key that started them; an admin may complete any key's program. 404 when
the caller holds no program with that id, which also covers programs completed earlier. A program none
of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed
response completes it with this feedback.
"""
routers: Final = _oracle_routers()
if not routers:
raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."})
found: Final = _find_program(routers, data.program_id, user_api_key_dict)
if found is None:
raise HTTPException(
status_code=404, detail={"error": f"Unknown or already completed program {data.program_id!r}."}
)
holder, binding = found
try:
holder.complete(
data.program_id, score=data.score, cost=data.cost, payload=data.payload, owner=binding.api_key_hash
)
except VerificationBacklogFull as error:
raise HTTPException(status_code=503, detail={"error": str(error)}) from error
return OracleRouterFeedbackResponse(
program_id=data.program_id,
router_name=holder.router_name,
model=binding.model,
pending_verifications=holder.pending_verifications,
deferred=binding.requests == 0,
)
@router.get(
"/oracle_router/state",
tags=["oracle_router"],
dependencies=[Depends(user_api_key_auth)],
response_model=OracleRouterStateResponse,
)
async def get_oracle_router_state(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> OracleRouterStateResponse:
"""Decision-maker state, active bindings and recent verified feedback for every ORACLE router. Admin only."""
if not _is_admin(user_api_key_dict):
raise HTTPException(status_code=403, detail={"error": CommonProxyErrors.not_allowed_access.value})
routers: Final = _oracle_routers()
if not routers:
raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."})
return OracleRouterStateResponse(routers=[await candidate.get_state_snapshot() for candidate in routers])

View file

@ -803,6 +803,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( # n
from litellm.proxy.management_endpoints.model_management_endpoints import (
router as model_management_router,
)
from litellm.proxy.management_endpoints.oracle_router_endpoints import (
router as oracle_router_management_router,
)
from litellm.proxy.management_endpoints.organization_endpoints import (
router as organization_router,
)
@ -22370,6 +22373,7 @@ app.include_router(budget_management_router)
app.include_router(model_management_router)
app.include_router(model_access_group_management_router)
app.include_router(auto_router_management_router)
app.include_router(oracle_router_management_router)
app.include_router(tag_management_router)
app.include_router(workflow_management_router)
app.include_router(memory_router)

View file

@ -62,6 +62,7 @@ if TYPE_CHECKING:
from litellm.router_strategy.auto_router.auto_router import AutoRouter
from litellm.router_strategy.budget_limiter import RouterBudgetLimiting
from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
from litellm.router_strategy.quality_router.quality_router import QualityRouter
from litellm.router_utils.cooldown_cache import CooldownCache
from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets as _AttemptedFallbackTargets
@ -104,6 +105,7 @@ if TYPE_CHECKING:
| dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]]
| dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]]
| dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]
| dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]]
| list[RoutingPlugin]
| RouterBudgetLimiting
| DualCache
@ -166,6 +168,7 @@ PluginKind: TypeAlias = Literal[
"complexity",
"quality",
"adaptive",
"oracle",
"routing",
"budget_limiter",
"prompt_caching_pins",
@ -1001,6 +1004,11 @@ class RouterRegistry:
self, kind: Literal["adaptive"]
) -> "dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]": ... # mutable-ok: the live dict
@overload
def plugins(
self, kind: Literal["oracle"]
) -> "dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]]": ... # mutable-ok: the live dict
@overload
def plugins(self, kind: Literal["routing"]) -> "list[RoutingPlugin]": ... # mutable-ok: the live list
@ -1012,7 +1020,7 @@ class RouterRegistry:
def plugins(self, kind: PluginKind) -> "RegistryPlugins":
"""
Plugins configured by the deployments: ``auto``, ``complexity``, ``quality`` and ``adaptive``
Plugins configured by the deployments: ``auto``, ``complexity``, ``quality``, ``adaptive`` and ``oracle``
sub-routers by deployment name, ``routing`` plugins, the deployment ``budget_limiter`` and
the ``prompt_caching_pins`` store (were ``auto_routers``, ``complexity_routers``,
``quality_routers``, ``adaptive_routers``, ``routing_plugins``,
@ -1027,6 +1035,8 @@ class RouterRegistry:
return r._core_quality_routers
if kind == "adaptive":
return r._core_adaptive_routers
if kind == "oracle":
return r._core_oracle_routers
if kind == "routing":
return r._core_routing_plugins
if kind == "budget_limiter":

View file

@ -448,6 +448,7 @@ from litellm.types.llms.openai import (
from litellm.types.router import (
CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS,
VALID_LITELLM_ENVIRONMENTS,
AdaptiveRouterPreferences,
AlertingConfig,
AllowedFailsPolicy,
AssistantsTypedDict,
@ -552,6 +553,7 @@ if TYPE_CHECKING:
from litellm.router_strategy.complexity_router.complexity_router import (
ComplexityRouter,
)
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
from litellm.router_strategy.quality_router.quality_router import (
QualityRouter,
)
@ -573,6 +575,7 @@ else:
ComplexityRouter = Any
AdaptiveRouter = Any
QualityRouter = Any
OracleRouter = Any # rebind-ok: runtime stand-in for the TYPE_CHECKING-only import above
PreRoutingHookResponse = Any
RouterStrategySelector: TypeAlias = (
@ -1812,6 +1815,7 @@ class PythonCore(DeprecatedRouterMembers):
self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] = {}
self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] = {}
self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] = {}
self._core_oracle_routers: dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]] = {} # mutable-ok: registry
self._core_routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else []
# Initialize model_group_alias early since it's used in set_model_list
@ -2111,6 +2115,7 @@ class PythonCore(DeprecatedRouterMembers):
_routers_holding_global_callbacks.discard(self)
slack_alerting_loggers: Final = self._slack_alerting_loggers()
adaptive_router_hooks: Final = self._adaptive_router_hooks()
oracle_router_hooks: Final = self._oracle_router_hooks()
vacated_keys: Final = self._host.callbacks.drop_callbacks(
(
*self._callback_selectors(),
@ -2118,6 +2123,7 @@ class PythonCore(DeprecatedRouterMembers):
*(self.optional_callbacks or ()),
*slack_alerting_loggers,
*adaptive_router_hooks,
*oracle_router_hooks,
),
method_owners=(self, *slack_alerting_loggers),
)
@ -2130,9 +2136,17 @@ class PythonCore(DeprecatedRouterMembers):
adaptive_survivor: Final = next((router for router in survivors if router._core_adaptive_routers), None)
if adaptive_survivor is not None and vacated_keys & adaptive_keys:
adaptive_survivor._sync_adaptive_router_hooks()
oracle_keys: Final = frozenset(map(self._host.callbacks.get_custom_logger_key, oracle_router_hooks))
oracle_survivor: Final = next((router for router in survivors if router._core_oracle_routers), None)
if oracle_survivor is not None and vacated_keys & oracle_keys:
oracle_survivor._sync_oracle_router_hooks()
def _core_serve_through_request_logging(self) -> None:
self._request_scoped_selectors = (*self._callback_selectors(), *self._request_adaptive_router_hooks())
self._request_scoped_selectors = (
*self._callback_selectors(),
*self._request_adaptive_router_hooks(),
*self._request_oracle_router_hooks(),
)
self._request_scoped_policies = (*self._optional_loggers(), *self._slack_alerting_loggers())
self._release_global_callbacks()
@ -2170,6 +2184,27 @@ class PythonCore(DeprecatedRouterMembers):
for tagged in chain.from_iterable(self._core_adaptive_routers.values())
)
def _request_oracle_router_hooks(self) -> tuple[CustomLogger, ...]:
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
return tuple(
OracleRouterPostCallHook(oracle_router=tagged.strategy)
for tagged in chain.from_iterable(self._core_oracle_routers.values())
)
def _oracle_router_hooks(self) -> tuple[CustomLogger, ...]:
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
own_strategy_ids: Final = frozenset(
id(tagged.strategy) for tagged in chain.from_iterable(self._core_oracle_routers.values())
)
hooks: Final = self._host.callbacks.get_custom_loggers_for_type(OracleRouterPostCallHook)
return tuple(
hook
for hook in hooks
if isinstance(hook, OracleRouterPostCallHook) and id(hook.oracle_router) in own_strategy_ids
)
def _routing_callbacks(self) -> Iterable[object]:
if not (self._request_scoped_selectors or self._request_scoped_policies):
return self._host.callback_list("callbacks")
@ -11543,6 +11578,8 @@ class PythonCore(DeprecatedRouterMembers):
self._unregister_pre_routing_strategy(registry, model_name, tags)
if self._unregister_pre_routing_strategy(self._core_adaptive_routers, model_name, tags):
self._sync_adaptive_router_hooks()
if self._unregister_pre_routing_strategy(self._core_oracle_routers, model_name, tags):
self._sync_oracle_router_hooks()
def _finalize_adaptive_router_if_configured(self) -> None:
"""Locate every adaptive-router deployment in the finalized model_list and
@ -11675,6 +11712,107 @@ class PythonCore(DeprecatedRouterMembers):
len(config.available_models),
)
def _is_oracle_router_deployment(self, litellm_params: LiteLLM_Params) -> bool:
return classify_strategy_router_model(litellm_params.model) == "oracle"
def _finalize_oracle_router_if_configured(self) -> None:
"""Deferred like the adaptive router, since a `pre_routing` decision maker wraps a strategy router that
may appear later in the list. Idempotent across hot reloads, so a router that already holds program
bindings and learned state is never rebuilt."""
for model_id in self._core_get_model_ids():
deployment = self._core_get_deployment(model_id=model_id)
if deployment is None or not self._is_oracle_router_deployment(litellm_params=deployment.litellm_params):
continue
if self._has_registered_strategy(
self._core_oracle_routers, deployment.model_name, self._deployment_tags(deployment)
):
continue
self.init_oracle_router_deployment(deployment=deployment)
self._sync_oracle_router_hooks()
def _sync_oracle_router_hooks(self) -> None:
"""Exactly one post-call hook per registered oracle router."""
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
for callback in self._host.callbacks.get_custom_loggers_for_type(OracleRouterPostCallHook):
self._host.callbacks.remove_callback_from_all_lists(callback)
for tagged in chain.from_iterable(self._core_oracle_routers.values()):
self._host.callbacks.add_litellm_callback(OracleRouterPostCallHook(oracle_router=tagged.strategy))
def _resolve_strategy_router(self, model_name: str) -> "PreRoutingStrategy | None":
"""The first strategy router registered under `model_name`, for ORACLE's `pre_routing` decision maker."""
for registry in (
self._core_complexity_routers,
self._core_quality_routers,
self._core_adaptive_routers,
self._core_auto_routers,
):
tagged_routers = registry.get(model_name)
if tagged_routers:
return tagged_routers[0].strategy
return None
def _deployment_input_costs(self, model_names: Sequence[str]) -> Mapping[str, float]:
"""``input_cost_per_token`` of the first deployment of each model name; litellm_params wins over model_info."""
named: Final = (
(name, self._core_get_deployment_by_model_group_name(model_group_name=name)) for name in model_names
)
return MappingProxyType(
{
name: deployment.model_info.input_cost_per_token
for name, deployment in named
if deployment is not None and deployment.model_info.input_cost_per_token is not None
}
)
def _declared_adaptive_prefs(self, model_name: str) -> AdaptiveRouterPreferences | None:
deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name)
extra: Final = deployment.model_info.model_extra if deployment is not None else None
raw: Final = extra.get("adaptive_router_preferences") if extra else None
return None if raw is None else AdaptiveRouterPreferences.model_validate(raw)
def _deployment_adaptive_prefs(self, model_names: Sequence[str]) -> Mapping[str, AdaptiveRouterPreferences]:
"""``model_info.adaptive_router_preferences`` of the first deployment of each model name, where declared."""
declared: Final = ((name, self._declared_adaptive_prefs(name)) for name in model_names)
return MappingProxyType({name: prefs for name, prefs in declared if prefs is not None})
def init_oracle_router_deployment(self, deployment: Deployment) -> None:
"""Build an OracleRouter from `oracle_router_config` and register it under `deployment.model_name`."""
from litellm.router_strategy.oracle_router.decision import build_decision_maker
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
from litellm.router_strategy.oracle_router.verifier import build_verifier
from litellm.types.router import OracleRouterConfig
raw_config: Final = deployment.litellm_params.oracle_router_config
if raw_config is None:
raise ValueError("oracle_router_config is required for oracle-router deployments.")
config: Final = OracleRouterConfig.model_validate(raw_config)
if deployment.model_info.db_model and "custom" in (config.decision_maker.type, config.verifier.type):
# A custom decision maker or verifier imports and runs Python named by the deployment. Only the
# operator's config file may name it; a model written through the API (any role) must not.
raise ValueError(
f"oracle_router deployment {deployment.model_name!r}: custom decision makers and verifiers "
"can only be configured in the proxy config file, not through the model API."
)
oracle_router: Final = OracleRouter(
router_name=deployment.model_name,
config=config,
decision_maker=build_decision_maker(
config.decision_maker,
tuple(config.available_models),
self._deployment_input_costs(config.available_models),
self._resolve_strategy_router,
self._deployment_adaptive_prefs(config.available_models),
),
verifier=build_verifier(config.verifier),
)
self._register_pre_routing_strategy(
registry=self._core_oracle_routers,
deployment=deployment,
strategy=oracle_router,
strategy_label="Oracle-router",
)
def _is_quality_router_deployment(self, litellm_params: LiteLLM_Params) -> bool:
"""
Check if the deployment is a quality-router deployment.
@ -11829,6 +11967,7 @@ class PythonCore(DeprecatedRouterMembers):
# Deferred: build the AdaptiveRouter strategy now that all underlying
# deployments have been registered.
self._finalize_adaptive_router_if_configured()
self._finalize_oracle_router_if_configured()
def _add_deployment(self, deployment: Deployment) -> Deployment:
import os
@ -12214,6 +12353,8 @@ class PythonCore(DeprecatedRouterMembers):
)
):
self._finalize_adaptive_router_if_configured()
if self._is_oracle_router_deployment(litellm_params=deployment.litellm_params):
self._finalize_oracle_router_if_configured()
return deployment
except Exception as e:
if self.ignore_invalid_deployments:
@ -16672,6 +16813,7 @@ class PythonCore(DeprecatedRouterMembers):
self._core_complexity_routers,
self._core_adaptive_routers,
self._core_quality_routers,
self._core_oracle_routers,
)
if not any(registries):
return None
@ -16773,6 +16915,7 @@ class PythonCore(DeprecatedRouterMembers):
self._core_complexity_routers,
self._core_adaptive_routers,
self._core_quality_routers,
self._core_oracle_routers,
)
):
return registered_model_name

View file

@ -26,7 +26,7 @@ from litellm.router_strategy.complexity_router.config import (
AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/"
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"]
StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality", "oracle"]
StrategyRouterDependencyRole: TypeAlias = Literal[
"tier", "default", "classifier", "embedding", "evaluation", "compactor"
@ -53,6 +53,7 @@ STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset(
"adaptive_router_config",
"quality_router_config",
"quality_router_default_model",
"oracle_router_config",
}
)
@ -65,6 +66,7 @@ _REQUIRED_FIELD_GROUPS: Final[Mapping[StrategyRouterKind, tuple[tuple[str, ...],
"complexity": (("complexity_router_config", "complexity_router_default_model"),),
"adaptive": (("adaptive_router_config",),),
"quality": (("quality_router_config", "quality_router_default_model"),),
"oracle": (("oracle_router_config",),),
}
@ -84,6 +86,8 @@ def classify_strategy_router_model(model: str) -> StrategyRouterKind | None:
return "adaptive"
if remainder.startswith("quality_router"):
return "quality"
if remainder.startswith("oracle_router"):
return "oracle"
return "semantic"
@ -143,6 +147,8 @@ def strategy_router_dependencies(
)
if kind == "adaptive":
return _pool(_mapping(litellm_params.get("adaptive_router_config")).get("available_models"), "tier")
if kind == "oracle":
return _pool(_mapping(litellm_params.get("oracle_router_config")).get("available_models"), "tier")
if kind == "quality":
quality: Final = _mapping(litellm_params.get("quality_router_config"))
return tuple(

View file

@ -0,0 +1,101 @@
# ORACLE router
Program-level routing for agentic workloads: an agentic task (a *program*) is bound to one model at its
first request, every later request with the same `program_id` reuses that binding, and the model is
chosen by a pluggable decision maker that learns from a pluggable verifier's delayed verdict.
Paper: https://arxiv.org/abs/2607.22465
## Config
```yaml
model_list:
- model_name: oracle-router
litellm_params:
model: auto_router/oracle_router
oracle_router_config:
available_models: ["smart", "fast"]
decision_maker: { type: thompson, weights: { quality: 0.7, cost: 0.3 } }
verifier: { type: guardrail, guardrail: answer-judge }
```
Clients send `metadata.program_id` (or `litellm_session_id`, or the `x-litellm-program-id` header)
with every request of a task. Requests without any id are routed statelessly and never learn. Concurrent
first requests of one program wait for the decision in flight, so a program is bound exactly once.
## Decision makers
| `type` | binds a program by | learns |
|---|---|---|
| `thompson` (default) | LiteLLM's adaptive-router bandit: a Beta posterior per (request type, model), Thompson sampling, cost-weighted pick with `weights: {quality, cost}` | from verified program scores |
| `pre_routing` | the strategy router named by `router` (complexity, quality, adaptive or semantic), once per program | no |
| `fixed` | always `model` | no |
| `custom` | `path: pkg.mod:Factory`, called with the model tuple; must expose `models`, `async select`, `update`, `snapshot` | up to you |
## Verifiers
| `type` | scores a finished program by |
|---|---|
| `guardrail` | applying the LiteLLM guardrail named `guardrail` to the program's final turn, with the transcript as `structured_messages` and the feedback `payload` as `request_data["metadata"]`: pass = 1, blocked = 0 (read from the status the guardrail records, so an `llm_as_a_judge` with `on_failure: log` still scores 0 when the answer fails). A guardrail that could not evaluate the program, for example a judge whose model was unreachable, counts as a failed verification and teaches the decision maker nothing |
| `reported` (default) | `metadata.program_score` on the last request, or `POST /oracle_router/feedback` |
| `custom` | `path: pkg.mod:Factory`, called with no arguments; must expose `async verify(outcome)` |
A verified score `s` adds `s` successes and `1 - s` failures to the cell of the model that served the
program. Updates are applied in verification-arrival order. A verifier that raises, or returns a score
that is not a finite number, counts as a failed verification (`verifications_failed` in the state) and
never reaches the decision maker.
## Finishing a program
Either mark the last request with `metadata.program_done: true` (plus `program_score` for the reported
verifier), or call `POST /oracle_router/feedback {"program_id": ..., "score": ..., "cost": ...}` from
the harness once the task is graded. Feedback that arrives before any response of the program was
observed (it raced the last response's logging, or every request failed) is deferred: the reply says
`deferred: true` and the next observed response completes the program with it, so nothing is ever
verified or learned without an answer. A guardrail verifier also fails the verification of a program
whose final response has no text, rather than reading a judge's skip as a pass. `GET /oracle_router/state`
(admin) shows bindings, decision-maker state and recent feedback. The response header
`x-litellm-oracle-router-model` names the bound model.
Verification runs in the background and is bounded: at most `max_pending_verifications` (default 64)
verifications are in flight per router, and each one is given `verification_timeout_seconds` (default
120) before it counts as failed. While the backlog is full a completion is refused and the program stays
bound, so it can be completed again once the backlog drains: the feedback endpoint answers 503 and a
`program_done` request records the refusal in `last_failure`; `verifications_rejected` in the state
counts them.
## Security
A program is private to the API key that started it: the same `program_id` under another key is another
program, so no caller can ride on, observe or complete someone else's binding. `POST /oracle_router/feedback`
completes the caller's own program; an admin key may complete any key's. The post-call hook only reads
requests the Router stamped as routed by this router (`routing_decision`), so ORACLE keys placed in the
metadata of a request to some other model are ignored, and the router drops any caller-supplied copy of
its internal metadata keys before stamping its own.
A `pre_routing` decision maker hands the strategy router it wraps what LiteLLM forwards to any detached
sub-call: the caller's identity (`user_api_key`, its hash and alias, team, user, organization and end user,
and the auth object the Router filters deployments by, minus the request's budget reservation), nothing else
of the request, and the program's `turn_off_message_logging` opt-out. A classifier call made on the program's
behalf is therefore billed to the caller's key and team and never logs a prompt the caller marked private;
the identity is used for that call only and is not kept on the program. Verification runs under the
program's opt-out as well: a program is private once any of its requests opted out, whether or not that
request produced an answer, so a `guardrail` verifier's judge call logs neither the prompt nor the
transcript. LiteLLM redacts a private request's messages and answer before the router's hook sees them, so
a `guardrail` verifier cannot judge a program whose final turn was redacted: that is a failed verification
and teaches the decision maker nothing. Use the `reported` verifier for routers whose programs opt out of
logging. A `guardrail` verifier's judge call carries no caller identity, as the proxy's own
`llm_as_a_judge` flow does not, so its spend is not attributed to the program's key or team. The feedback
`payload` reaches the guardrail with the proxy-owned keys removed (`user_api_key_*`, the sub-call origin,
headers and the routing decision), so a caller cannot pose as the proxy to a guardrail that forwards
identity to its vendor.
`custom` decision makers and verifiers import Python named by the deployment, so they are honoured only
for deployments in the proxy config file. A model written through the API (`/model/new`, `/model/update`,
whatever the caller's role) with a `custom` type is rejected when the router is built and never imported.
## Limits
State is in memory per proxy process; it is not persisted or shared across replicas. The paper's
dispatch scheduler (DISC) is not part of this strategy. The router and its hook write nothing to the
process log: `GET /oracle_router/state` carries the counters and the last background failure
(`last_failure`) instead.

View file

@ -0,0 +1,32 @@
from litellm.router_strategy.oracle_router.decision import (
DecisionMaker,
FixedDecisionMaker,
PreRoutingDecisionMaker,
ProgramContext,
ThompsonDecisionMaker,
build_decision_maker,
)
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, ProgramBinding
from litellm.router_strategy.oracle_router.verifier import (
GuardrailVerifier,
ProgramOutcome,
ReportedVerifier,
Verifier,
build_verifier,
)
__all__ = [
"DecisionMaker",
"FixedDecisionMaker",
"GuardrailVerifier",
"OracleRouter",
"PreRoutingDecisionMaker",
"ProgramBinding",
"ProgramContext",
"ProgramOutcome",
"ReportedVerifier",
"ThompsonDecisionMaker",
"Verifier",
"build_decision_maker",
"build_verifier",
]

View file

@ -0,0 +1,18 @@
"""Metadata keys and headers shared by the ORACLE router, its post-call hook and the proxy endpoints."""
from typing import Final
ORACLE_ROUTER_MODEL_PREFIX: Final[str] = "auto_router/oracle_router"
CHOSEN_MODEL_METADATA_KEY: Final[str] = "oracle_router_chosen_model"
PROGRAM_ID_METADATA_KEY: Final[str] = "oracle_router_program_id"
RESPONSE_HEADER: Final[str] = "x-litellm-oracle-router-model"
PROGRAM_ID_HEADER: Final[str] = "x-litellm-program-id"
PROGRAM_DONE_KEY: Final[str] = "program_done"
PROGRAM_SCORE_KEY: Final[str] = "program_score"
SESSION_ID_FALLBACK_KEYS: Final[tuple[str, ...]] = ("litellm_session_id", "session_id")
FEEDBACK_HISTORY_SIZE: Final[int] = 200
PROGRAM_SWEEP_THRESHOLD: Final[int] = 1024

View file

@ -0,0 +1,248 @@
"""Decision makers: the pluggable policy that binds a program to a model.
``DecisionMaker`` is the whole contract. The default, ``thompson``, is LiteLLM's own adaptive-router
bandit (a Beta posterior per request type and model, Thompson sampling, cost-weighted pick) trained on
verified program outcomes instead of per-turn regex signals. ``pre_routing`` lets any LiteLLM strategy
router (complexity, quality, adaptive, semantic) make the proposal once per program, ``fixed`` pins one
model, and ``custom`` imports an operator's class.
"""
import importlib
import itertools
import random
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final, Protocol, runtime_checkable
from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy
from litellm.router_strategy.adaptive_router.bandit import BanditCell, apply_delta, initial_cell, pick_best
from litellm.types.router import (
AdaptiveRouterPreferences,
AdaptiveRouterWeights,
OracleDecisionMakerConfig,
PreRoutingStrategy,
RequestType,
)
_NO_CALLER: Final[Mapping[str, object]] = MappingProxyType({})
@dataclass(frozen=True, slots=True)
class ProgramContext:
"""What a decision maker sees when a program arrives: its initial request and LiteLLM's request type."""
program_id: str
prompt: str
request_type: RequestType
message_logging_off: bool = False
caller: Mapping[str, object] = field(default_factory=lambda: _NO_CALLER)
"""The caller's identity metadata, so a sub-call made on the program's behalf is billed to the caller."""
@runtime_checkable
class DecisionMaker(Protocol):
@property
def models(self) -> tuple[str, ...]: ...
async def select(self, context: ProgramContext) -> str: ...
def update(self, context: ProgramContext, model: str, score: float) -> None: ...
def snapshot(self) -> Mapping[str, object]: ...
def _check_model(models: Sequence[str], model: str) -> None:
if model not in models:
raise KeyError(f"unknown model {model!r}; configured models: {list(models)}")
_UNIFORM_PRIOR: Final = AdaptiveRouterPreferences(quality_tier=2, strengths=[])
class ThompsonDecisionMaker:
"""LiteLLM's adaptive-router bandit, keyed by (request type, model) and updated with verified scores.
Cells start from the same priors the adaptive router uses, read from each model's
``model_info.adaptive_router_preferences`` (tier 2 when a model declares none). A verified score ``s``
in [0, 1] adds ``s`` successes and ``1 - s`` failures to the cell of the model that served the program.
Selection samples every cell once and picks the best quality/cost score with the adaptive router's
weights, so ``weights.cost`` is the only cost knob.
"""
def __init__(
self,
models: Sequence[str],
model_costs: Mapping[str, float],
weights: AdaptiveRouterWeights | None = None,
model_prefs: Mapping[str, AdaptiveRouterPreferences] | None = None,
seed: int = 0,
) -> None:
if not models:
raise ValueError("a decision maker needs at least one model")
self._models: Final[tuple[str, ...]] = tuple(models)
costs: Final = {m: float(model_costs.get(m, 0.0)) for m in self._models}
self._costs: Final[dict[str, float]] = costs # mutable-ok: pick_best takes a dict
self._weights: Final[AdaptiveRouterWeights] = weights or AdaptiveRouterWeights()
self._rng: Final = random.Random(seed)
prefs: Final = model_prefs or {}
grid: Final = tuple(itertools.product(RequestType, self._models))
cells: Final = {(rt, model): initial_cell(prefs.get(model, _UNIFORM_PRIOR), rt) for rt, model in grid}
self._cells: Final[dict[tuple[RequestType, str], BanditCell]] = cells # mutable-ok: replaced on update
self.pulls: Final[dict[str, int]] = {m: 0 for m in self._models} # mutable-ok: counters
@property
def models(self) -> tuple[str, ...]:
return self._models
async def select(self, context: ProgramContext) -> str:
cells: Final = {model: self._cells[(context.request_type, model)] for model in self._models}
chosen: Final = pick_best(cells, self._costs, self._weights.quality, self._weights.cost, rng=self._rng)
self.pulls[chosen] += 1
return chosen
def update(self, context: ProgramContext, model: str, score: float) -> None:
_check_model(self._models, model)
key: Final = (context.request_type, model)
self._cells[key] = apply_delta(self._cells[key], score, 1.0 - score)
def estimate(self, context: ProgramContext, model: str) -> float:
_check_model(self._models, model)
return self._cells[(context.request_type, model)].mean
def snapshot(self) -> Mapping[str, object]:
cells: Final = {
f"{request_type.value}/{model}": {"mean": round(cell.mean, 4), "samples": cell.total_samples}
for (request_type, model), cell in self._cells.items()
if cell.total_samples > 0
}
return {
"type": "thompson",
"weights": {"quality": self._weights.quality, "cost": self._weights.cost},
"pulls": dict(self.pulls),
"cells": cells,
}
class FixedDecisionMaker:
"""Always the same model; the single-model baseline that still gets verified and reported."""
def __init__(self, models: Sequence[str], model: str) -> None:
_check_model(models, model)
self._models: Final[tuple[str, ...]] = tuple(models)
self._model: Final[str] = model
self._updates: int = 0
@property
def models(self) -> tuple[str, ...]:
return self._models
async def select(self, context: ProgramContext) -> str:
return self._model
def update(self, context: ProgramContext, model: str, score: float) -> None:
_check_model(self._models, model)
self._updates += 1
def snapshot(self) -> Mapping[str, object]:
return {"type": "fixed", "model": self._model, "updates": self._updates}
def strategy_request_kwargs(context: ProgramContext) -> dict[str, object]: # mutable-ok: pre-routing contract
"""The request a wrapped strategy router sees for the program: the caller's identity, so a classifier call
the strategy makes is billed to the caller's key and team, and the program's message-logging opt-out where
LiteLLM reads it. A fresh dict per call: the strategies write their decision into the bucket they are handed."""
privacy: Final[Mapping[str, object]] = {"turn_off_message_logging": True} if context.message_logging_off else {}
return {"metadata": {**context.caller, **privacy}}
class PreRoutingDecisionMaker:
"""Let one of LiteLLM's own strategy routers propose the model, once per program.
The wrapped router (complexity, quality, adaptive or semantic) sees the program's initial request as a
single user message. Its per-request decision becomes a per-program binding, and the program's verified
outcome is recorded against that model, so the state endpoint shows what the strategy actually achieved.
"""
def __init__(self, models: Sequence[str], router_name: str, strategy: PreRoutingStrategy) -> None:
if not models:
raise ValueError("a decision maker needs at least one model")
self._models: Final[tuple[str, ...]] = tuple(models)
self._router_name: Final[str] = router_name
self._strategy: Final[PreRoutingStrategy] = strategy
self._proposals: Final[dict[str, int]] = {} # mutable-ok: proposal counts per model, incremented per program
self._fallbacks: int = 0
self._updates: int = 0
@property
def models(self) -> tuple[str, ...]:
return self._models
async def select(self, context: ProgramContext) -> str:
with inherit_message_logging_privacy(context.message_logging_off):
response: Final = await self._strategy.async_pre_routing_hook(
model=self._router_name,
request_kwargs=strategy_request_kwargs(context),
messages=[{"role": "user", "content": context.prompt}],
)
proposed: Final = response.model if response is not None and response.model in self._models else None
if proposed is None:
self._fallbacks += 1
chosen: Final = proposed if proposed is not None else self._models[0]
self._proposals[chosen] = self._proposals.get(chosen, 0) + 1
return chosen
def update(self, context: ProgramContext, model: str, score: float) -> None:
_check_model(self._models, model)
self._updates += 1
def snapshot(self) -> Mapping[str, object]:
return {
"type": "pre_routing",
"router": self._router_name,
"proposals": dict(self._proposals),
"fallbacks": self._fallbacks,
"updates": self._updates,
}
def load_object(path: str) -> object:
"""Import ``package.module:attribute``."""
module_name, separator, attribute = path.partition(":")
if not separator or not module_name or not attribute:
raise ValueError(f"expected 'package.module:Attribute', got {path!r}")
loaded: Final[object] = getattr(importlib.import_module(module_name), attribute) # pyright: ignore[reportAny] # module attributes are untyped
return loaded
def build_decision_maker(
config: OracleDecisionMakerConfig,
models: Sequence[str],
model_costs: Mapping[str, float],
resolve_strategy: Callable[[str], PreRoutingStrategy | None],
model_prefs: Mapping[str, AdaptiveRouterPreferences] | None = None,
) -> DecisionMaker:
"""Instantiate the configured decision maker over ``models`` (strongest first)."""
match config.type:
case "thompson":
return ThompsonDecisionMaker(
models, model_costs, weights=config.weights, model_prefs=model_prefs, seed=config.seed
)
case "fixed":
return FixedDecisionMaker(models, model=config.model or models[0])
case "pre_routing":
router_name: Final = config.router or ""
strategy: Final = resolve_strategy(router_name)
if strategy is None:
raise ValueError(
f"oracle_router decision_maker.router {router_name!r} is not a configured strategy router"
)
return PreRoutingDecisionMaker(models, router_name=router_name, strategy=strategy)
case "custom":
factory: Final = load_object(config.path or "")
if not callable(factory):
raise TypeError(f"oracle_router decision_maker.path {config.path!r} is not callable")
instance: Final = factory(tuple(models))
if not isinstance(instance, DecisionMaker):
raise TypeError(f"{config.path!r} must build an object with select, update, snapshot and models")
return instance

View file

@ -0,0 +1,119 @@
"""Post-call hook for the ORACLE router.
After every completion it adds LiteLLM's computed spend and the latest answer to the program's row, and
when the request carried ``metadata.program_done`` it completes the program: the slot is released at once
and the verifier runs in the background. Only requests the Router stamped as routed by this ORACLE router
are read, and only against the program of the key that sent them. Nothing here can fail a request, and
nothing here logs: a failure is kept on the router for ``/oracle_router/state``.
"""
import math
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final
from litellm.integrations.custom_logger import CustomLogger
from litellm.router_strategy.oracle_router.config import (
CHOSEN_MODEL_METADATA_KEY,
PROGRAM_DONE_KEY,
PROGRAM_ID_METADATA_KEY,
PROGRAM_SCORE_KEY,
RESPONSE_HEADER,
)
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, as_str_mapping, logged_request_is_private
from litellm.router_strategy.oracle_router.verifier import response_text
if TYPE_CHECKING:
from litellm.proxy._types import UserAPIKeyAuth
def _request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]:
"""The request's metadata as the proxy saw it: the caller's bucket with the proxy-internal one on top."""
litellm_params: Final = as_str_mapping(kwargs.get("litellm_params"))
return _merged(litellm_params.get("metadata"), litellm_params.get("litellm_metadata"))
def _merged(metadata: object, litellm_metadata: object) -> Mapping[str, object]:
return {**as_str_mapping(metadata), **as_str_mapping(litellm_metadata)}
def _routed_by(metadata: Mapping[str, object], router_name: str) -> bool:
"""True when the Router stamped this request as routed by this ORACLE router.
The Router writes or clears ``routing_decision`` on every routing attempt, so a caller cannot make a
request to some other model look like one of this router's by putting ORACLE keys in its metadata.
"""
decision: Final = as_str_mapping(metadata.get("routing_decision"))
return decision.get("router_type") == "oracle" and decision.get("router_model_name") == router_name
def _owner(metadata: Mapping[str, object]) -> str | None:
"""The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy."""
value: Final = metadata.get("user_api_key_hash")
return str(value) if value else None
def _score(metadata: Mapping[str, object]) -> float | None:
raw: Final = metadata.get(PROGRAM_SCORE_KEY)
if isinstance(raw, bool) or not isinstance(raw, (int, float, str)):
return None
try:
score: Final = float(raw)
except ValueError:
return None
return score if math.isfinite(score) else None # a NaN would corrupt a bandit cell for good
class OracleRouterPostCallHook(CustomLogger):
"""One hook per OracleRouter, registered in ``litellm.callbacks``."""
def __init__(self, oracle_router: OracleRouter) -> None:
self.oracle_router: Final[OracleRouter] = oracle_router
async def async_post_call_response_headers_hook(
self,
data: Mapping[str, object],
user_api_key_dict: "UserAPIKeyAuth",
response: object,
request_headers: dict[str, str] | None = None, # mutable-ok: CustomLogger's hook signature
litellm_call_info: dict[str, object] | None = None, # mutable-ok: CustomLogger's hook signature
) -> dict[str, str] | None: # mutable-ok: CustomLogger's hook signature
metadata: Final = _merged(data.get("metadata"), data.get("litellm_metadata"))
if not _routed_by(metadata, self.oracle_router.router_name):
return None
chosen: Final = metadata.get(CHOSEN_MODEL_METADATA_KEY)
return {RESPONSE_HEADER: str(chosen)} if chosen else None
async def async_log_success_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record(kwargs, response_obj, succeeded=True)
async def async_log_failure_event(
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object
) -> None:
self._record(kwargs, response_obj, succeeded=False)
def _record(self, kwargs: Mapping[str, object], response_obj: object, succeeded: bool) -> None:
try:
metadata: Final = _request_metadata(kwargs)
if not _routed_by(metadata, self.oracle_router.router_name):
return
program_id: Final = metadata.get(PROGRAM_ID_METADATA_KEY)
if not isinstance(program_id, str):
return
owner: Final = _owner(metadata)
if logged_request_is_private(kwargs):
self.oracle_router.mark_message_logging_off(program_id, owner)
if succeeded:
cost: Final = kwargs.get("response_cost")
self.oracle_router.observe_request(
program_id,
cost=float(cost) if isinstance(cost, (int, float)) else 0.0,
response_text=response_text(response_obj),
messages=kwargs.get("messages"),
owner=owner,
)
if metadata.get(PROGRAM_DONE_KEY):
self.oracle_router.complete(program_id, score=_score(metadata), owner=owner)
except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes
self.oracle_router.note_failure("post-call hook", error)

View file

@ -0,0 +1,548 @@
"""The ORACLE router: one binding per agentic program, learned from verified, delayed feedback.
A program is one agentic task made of many LLM requests that share a prefix. Its first request is bound
to a model by the decision maker and every later request with the same program id reuses that binding.
When the program finishes, the verifier scores it off the critical path, the score and LiteLLM's spend
for the program become one reward, and the decision maker learns from it in arrival order.
Paper: ORACLE, https://arxiv.org/abs/2607.22465
"""
import asyncio
import time
from collections import deque
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, replace
from types import MappingProxyType
from typing import Final, NamedTuple
from pydantic import TypeAdapter, ValidationError
from litellm.litellm_core_utils.core_helpers import (
get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict
)
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
inherit_message_logging_privacy,
initialize_standard_callback_dynamic_params,
)
from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
from litellm.litellm_core_utils.redact_messages import (
should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # upstream helper takes a bare dict
)
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
from litellm.router_strategy.oracle_router.config import (
CHOSEN_MODEL_METADATA_KEY,
FEEDBACK_HISTORY_SIZE,
PROGRAM_ID_HEADER,
PROGRAM_ID_METADATA_KEY,
PROGRAM_SWEEP_THRESHOLD,
SESSION_ID_FALLBACK_KEYS,
)
from litellm.router_strategy.oracle_router.decision import DecisionMaker, ProgramContext
from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, Verifier, clamp_score
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import OracleRouterConfig, PreRoutingHookResponse, RequestType
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
RoutingDecisionCause,
StandardLoggingRoutingDecision,
)
_LEARNING_DECISION_MAKERS: Final[frozenset[str]] = frozenset({"thompson"})
_MESSAGES: Final = TypeAdapter(Sequence[AllMessageValues])
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
_OBJECTS: Final = TypeAdapter(Sequence[object])
def as_str_mapping(value: object) -> Mapping[str, object]:
"""A typed, read-only view of a request-shaped mapping; empty for anything else."""
if not isinstance(value, Mapping):
return _EMPTY
try:
return _STR_MAPPING.validate_python(value)
except ValidationError:
return _EMPTY
class ProgramKey(NamedTuple):
"""A program is private to the API key that started it: the same id under another key is another program."""
owner: str | None
program_id: str
@dataclass(frozen=True, slots=True)
class DeferredCompletion:
"""A completion that arrived before any response of the program was observed; the next one applies it."""
score: float | None
cost: float | None
payload: Mapping[str, object]
@dataclass(frozen=True, slots=True)
class ProgramBinding:
"""One row of the lookup table: program id to model, plus what the proxy has seen of the program."""
program_id: str
model: str
context: ProgramContext
api_key_hash: str | None
bound_at: float
requests: int = 0
cost: float = 0.0
last_response_text: str = ""
last_messages: Sequence[AllMessageValues] = ()
deferred: DeferredCompletion | None = None
message_logging_off: bool = False
"""True once any request of the program opted out of message logging; verification then runs privately."""
@dataclass(frozen=True, slots=True)
class FeedbackRecord:
program_id: str
model: str
request_type: str
score: float
cost: float
verified_at: float
def program_messages(
messages: list[dict[str, object]] | None, # mutable-ok: pre-routing contract
input: str | list[object] | None, # mutable-ok: pre-routing contract
request_kwargs: Mapping[str, object],
) -> Sequence[Mapping[str, object]] | None:
"""The request's messages in chat shape whichever surface sent them: a Responses API ``input`` is converted
the way guardrails convert it, so a program started through ``/v1/responses`` is bound on its real prompt."""
if messages or input is None:
return messages
return resolve_structured_messages(
None,
{**request_kwargs, "input": input, "litellm_metadata": as_str_mapping(request_kwargs.get("litellm_metadata"))},
)
def first_user_text(messages: Sequence[Mapping[str, object]] | None) -> str:
"""The program's initial request: the first user message, text parts only."""
for message in messages or ():
if message.get("role") != "user":
continue
content = message.get("content")
if isinstance(content, str):
return content
if isinstance(content, Sequence):
parts = tuple(as_str_mapping(part) for part in _OBJECTS.validate_python(content))
return "\n".join(str(part.get("text", "")) for part in parts if part.get("type") == "text")
return ""
return ""
def _metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
"""The request's metadata as callers see it, with the proxy-internal bucket layered on top."""
return {**as_str_mapping(request_kwargs.get("metadata")), **as_str_mapping(request_kwargs.get("litellm_metadata"))}
def _header(request_kwargs: Mapping[str, object], name: str) -> str | None:
headers: Final = as_str_mapping(request_kwargs.get("headers"))
matches: Final = tuple(value for key, value in headers.items() if key.lower() == name)
return str(matches[0]) if matches else None
def resolve_program_id(request_kwargs: Mapping[str, object], program_id_key: str) -> str | None:
"""The program id of a request: ``metadata[program_id_key]``, then the session id spellings, then the header."""
metadata: Final = _metadata(request_kwargs)
candidates: Final = (
metadata.get(program_id_key),
*(metadata.get(key) for key in SESSION_ID_FALLBACK_KEYS),
request_kwargs.get("litellm_session_id"),
_header(request_kwargs, PROGRAM_ID_HEADER),
)
return next((str(candidate) for candidate in candidates if candidate), None)
def _transcript(messages: object, fallback: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]:
if not messages:
return fallback
try:
return tuple(_MESSAGES.validate_python(messages))
except ValidationError:
return fallback
def request_is_private(request_kwargs: Mapping[str, object]) -> bool:
"""Whether LiteLLM will redact this request's messages when it logs them, by the logger's own rule: the
request's ``turn_off_message_logging``, its redaction headers, or the global setting."""
kwargs: Final = dict(request_kwargs)
return _redacts(
{
"litellm_params": kwargs,
"standard_callback_dynamic_params": initialize_standard_callback_dynamic_params(kwargs),
}
)
def logged_request_is_private(kwargs: Mapping[str, object]) -> bool:
"""Whether LiteLLM redacted a logged request's messages, from the kwargs it hands a logging callback."""
return _redacts(
{
**kwargs,
"standard_callback_dynamic_params": kwargs.get("standard_callback_dynamic_params")
or initialize_standard_callback_dynamic_params(dict(kwargs)),
}
)
def _redacts(model_call_details: dict[str, object]) -> bool: # mutable-ok: the logger's own parameter shape
try:
return should_redact_message_logging(model_call_details)
except AttributeError: # malformed metadata makes the logger raise and log nothing, so the program stays private
return True
def forwardable_caller(request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
"""What a classifier call made on the program's behalf inherits from the request that bound it: the caller's
identity from whichever metadata bucket the surface used, with its budget reservation removed, stamped as the
sub-call it becomes. The routing decision, the logging payload and every other per-request state stay behind."""
return MappingProxyType(
sanitized_forwardable_call_metadata(_metadata(request_kwargs), AUTOROUTER_CLASSIFIER_CALL_ORIGIN)
)
def _api_key_hash(request_kwargs: Mapping[str, object]) -> str | None:
"""The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy."""
value: Final = _metadata(request_kwargs).get("user_api_key_hash")
return str(value) if value else None
def _stamp_internal(
request_kwargs: dict[str, object], # mutable-ok: pre-routing contract
key: str,
value: object | None,
) -> None:
"""Write a router-internal metadata key where the proxy keeps its own state.
Any copy the caller supplied is dropped from both buckets first, so the post-call hook only ever reads
what this router wrote; None clears the key.
"""
for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")):
if isinstance(bucket, dict):
bucket.pop(key, None) # pyright: ignore[reportUnknownMemberType] # caller-supplied dict
if value is not None:
_, internal = get_or_create_metadata_bucket(request_kwargs) # pyright: ignore[reportUnknownVariableType] # bare dict upstream
internal[key] = value
class VerificationBacklogFull(RuntimeError):
"""``max_pending_verifications`` verifications are in flight; the program stays bound for a later retry."""
class OracleRouter:
"""One instance per ``auto_router/oracle_router`` deployment."""
def __init__(
self,
router_name: str,
config: OracleRouterConfig,
decision_maker: DecisionMaker,
verifier: Verifier,
clock: Callable[[], float] = time.time,
) -> None:
self.router_name: Final[str] = router_name
self.config: Final[OracleRouterConfig] = config
self.decision_maker: Final[DecisionMaker] = decision_maker
self.verifier: Final[Verifier] = verifier
self._clock: Final = clock
self._cause: Final[RoutingDecisionCause] = (
"bandit" if config.decision_maker.type in _LEARNING_DECISION_MAKERS else "classifier_plugin"
)
self._bindings: Final[dict[ProgramKey, ProgramBinding]] = {} # mutable-ok: the lookup table
self._binding_in_flight: Final[dict[ProgramKey, asyncio.Event]] = {} # mutable-ok: being bound right now
self._tasks: Final[set[asyncio.Task[float]]] = set() # mutable-ok: in-flight verifications
self._history: Final[deque[FeedbackRecord]] = deque(maxlen=FEEDBACK_HISTORY_SIZE) # mutable-ok: bounded log
self._update_lock: Final = asyncio.Lock()
self.programs_bound: int = 0
self.programs_completed: int = 0
self.programs_evicted: int = 0
self.stateless_requests: int = 0
self.verifications_failed: int = 0
self.verifications_rejected: int = 0
self.last_failure: str | None = None
@property
def models(self) -> tuple[str, ...]:
return tuple(self.config.available_models)
def note_failure(self, step: str, error: Exception) -> None:
"""Keep the last failure of a background step for ``/oracle_router/state``.
The router and its hook write nothing to the process log: every request input reaches them, and a
log call on that path makes CodeQL's log-injection analysis fall back to a whole-codebase pass that
no longer fits its limits on this codebase. The admin state endpoint carries the failure instead.
"""
self.last_failure = f"{step}: {type(error).__name__}: {error}"
def context_for(self, program_id: str, prompt: str, request_kwargs: Mapping[str, object]) -> ProgramContext:
return ProgramContext(
program_id=program_id,
prompt=prompt,
request_type=classify_prompt(prompt),
message_logging_off=request_is_private(request_kwargs),
caller=forwardable_caller(request_kwargs),
)
def binding(self, program_id: str, owner: str | None = None) -> ProgramBinding | None:
"""The program's row as seen by ``owner``, the key that started it (``user_api_key_hash``)."""
return self._bindings.get(ProgramKey(owner, program_id))
def bindings_named(self, program_id: str) -> tuple[ProgramBinding, ...]:
"""Every key's program with this id, oldest first: an admin may complete another key's program."""
matching: Final = (binding for key, binding in self._bindings.items() if key.program_id == program_id)
return tuple(sorted(matching, key=lambda binding: binding.bound_at))
def _evict_expired(self, now: float) -> None:
deadline: Final = now - self.config.program_ttl_seconds
expired: Final = tuple(key for key, binding in self._bindings.items() if binding.bound_at < deadline)
for key in expired:
self._bindings.pop(key, None)
overflow: Final = len(self._bindings) + 1 - self.config.max_programs # +1: room for the program being bound
oldest: Final = tuple(sorted(self._bindings, key=lambda key: self._bindings[key].bound_at))[: max(overflow, 0)]
for key in oldest:
self._bindings.pop(key, None)
self.programs_evicted += len(expired) + len(oldest)
async def bind(
self, program_id: str, prompt: str, api_key_hash: str | None, request_kwargs: Mapping[str, object] = _EMPTY
) -> ProgramBinding:
"""Bind a new program, or return its existing row.
A program is private to the key that started it: the same id under another key starts that key's
own program instead of riding on, or completing, someone else's. Concurrent first requests of one
program (a harness that fans out) wait for the decision in flight instead of each asking the
decision maker, so a program is bound exactly once.
"""
key: Final = ProgramKey(api_key_hash, program_id)
in_flight = self._binding_in_flight.get(key) # rebind-ok: re-read after each wait
while in_flight is not None:
await in_flight.wait()
in_flight = self._binding_in_flight.get(key)
existing: Final = self._bindings.get(key)
if existing is not None:
return existing
event: Final = asyncio.Event()
self._binding_in_flight[key] = event
try:
return await self._bind(key, prompt, request_kwargs)
finally:
self._binding_in_flight.pop(key, None)
event.set()
async def _bind(self, key: ProgramKey, prompt: str, request_kwargs: Mapping[str, object]) -> ProgramBinding:
now: Final = self._clock()
if len(self._bindings) >= min(PROGRAM_SWEEP_THRESHOLD, self.config.max_programs):
self._evict_expired(now)
context: Final = self.context_for(key.program_id, prompt, request_kwargs)
model: Final = await self.decision_maker.select(context)
binding: Final = ProgramBinding(
program_id=key.program_id,
model=model,
context=replace(context, caller=_EMPTY),
api_key_hash=key.owner,
bound_at=now,
message_logging_off=context.message_logging_off,
)
self._bindings[key] = binding
self.programs_bound += 1
return binding
async def async_pre_routing_hook(
self,
model: str,
request_kwargs: dict[str, object], # mutable-ok: pre-routing contract
messages: list[dict[str, object]] | None = None, # mutable-ok: pre-routing contract
input: str | list[object] | None = None, # mutable-ok: pre-routing contract
specific_deployment: bool | None = False,
) -> PreRoutingHookResponse | None:
"""Route the request to its program's model, binding the program on its first request."""
program_id: Final = resolve_program_id(request_kwargs, self.config.program_id_key)
prompt: Final = first_user_text(program_messages(messages, input, request_kwargs))
chosen_model, request_type = await self._decide(program_id, prompt, request_kwargs)
_stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model)
_stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id)
return PreRoutingHookResponse(
model=chosen_model,
messages=messages,
routing_decision=StandardLoggingRoutingDecision(
router_model_name=self.router_name,
router_type="oracle",
routed_model=chosen_model,
cause=self._cause,
request_type=request_type.value,
),
)
async def _decide(
self, program_id: str | None, prompt: str, request_kwargs: Mapping[str, object]
) -> tuple[str, RequestType]:
if program_id is None:
self.stateless_requests += 1
context: Final = self.context_for("", prompt, request_kwargs)
return await self.decision_maker.select(context), context.request_type
binding: Final = await self.bind(program_id, prompt, _api_key_hash(request_kwargs), request_kwargs)
return binding.model, binding.context.request_type
def mark_message_logging_off(self, program_id: str, owner: str | None = None) -> None:
"""A request of the program opted out of message logging, whether or not it produced an answer: the
whole program is private from here on, and its verification runs under that opt-out."""
key: Final = ProgramKey(owner, program_id)
binding: Final = self._bindings.get(key)
if binding is not None and not binding.message_logging_off:
self._bindings[key] = replace(binding, message_logging_off=True)
def observe_request(
self, program_id: str, cost: float, response_text: str, messages: object = None, owner: str | None = None
) -> None:
"""Account one finished LLM request of the program: LiteLLM's spend, the latest answer and transcript."""
key: Final = ProgramKey(owner, program_id)
binding: Final = self._bindings.get(key)
if binding is None:
return
transcript: Final = _transcript(messages, binding.last_messages)
self._bindings[key] = replace(
binding,
requests=binding.requests + 1,
cost=binding.cost + max(cost, 0.0),
last_response_text=response_text or binding.last_response_text,
last_messages=transcript,
)
deferred: Final = binding.deferred
if deferred is not None: # the harness reported before this response was observed
self.complete(program_id, score=deferred.score, cost=deferred.cost, payload=deferred.payload, owner=owner)
def complete(
self,
program_id: str,
score: float | None = None,
cost: float | None = None,
payload: Mapping[str, object] = _EMPTY,
owner: str | None = None,
) -> asyncio.Task[float] | None:
"""Release the program now and verify it in the background; the task resolves to the verified score.
A program none of whose requests has been observed yet (feedback that raced the last response's
logging, or a program whose only request failed) keeps its slot and is completed, with these
arguments, by its next observed response: there is no answer to verify yet, so nothing is learned.
Raises ``VerificationBacklogFull`` when ``max_pending_verifications`` verifications are already in
flight: the program stays bound, so the caller can complete it again once the backlog drains.
"""
key: Final = ProgramKey(owner, program_id)
binding: Final = self._bindings.get(key)
if binding is None:
return None
if binding.requests == 0:
self._bindings[key] = replace(binding, deferred=DeferredCompletion(score=score, cost=cost, payload=payload))
return None
if len(self._tasks) >= self.config.max_pending_verifications:
self.verifications_rejected += 1
raise VerificationBacklogFull(
f"{len(self._tasks)} verifications are in flight (max_pending_verifications="
f"{self.config.max_pending_verifications}); program {program_id!r} stays bound, complete it again later"
)
del self._bindings[key]
outcome: Final = ProgramOutcome(
program_id=program_id,
model=binding.model,
prompt=binding.context.prompt,
response_text=binding.last_response_text,
cost=cost if cost is not None else binding.cost,
messages=binding.last_messages,
score=score,
payload=payload,
message_logging_off=binding.message_logging_off,
)
self.programs_completed += 1
task: Final = asyncio.create_task(self._verify_and_learn(binding, outcome))
self._tasks.add(task)
task.add_done_callback(self._tasks.discard)
return task
async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float:
try:
with inherit_message_logging_privacy(outcome.message_logging_off):
score: Final = clamp_score(
await asyncio.wait_for(
self.verifier.verify(outcome), timeout=self.config.verification_timeout_seconds
)
)
except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop
self.verifications_failed += 1
self.note_failure("verification", error)
return float("nan")
async with self._update_lock:
self.decision_maker.update(binding.context, binding.model, score)
self._history.append(
FeedbackRecord(
program_id=binding.program_id,
model=binding.model,
request_type=binding.context.request_type.value,
score=score,
cost=outcome.cost,
verified_at=self._clock(),
)
)
return score
async def drain(self) -> None:
"""Wait for every in-flight verification."""
while self._tasks:
await asyncio.gather(*tuple(self._tasks), return_exceptions=True)
@property
def pending_verifications(self) -> int:
return len(self._tasks)
def _accuracy_by_model(self) -> Mapping[str, float]:
scores: Final[dict[str, list[float]]] = {} # mutable-ok: grouping accumulator for the snapshot
for record in self._history:
scores.setdefault(record.model, []).append(record.score)
return {model: round(sum(values) / len(values), 4) for model, values in scores.items()}
async def get_state_snapshot(self) -> Mapping[str, object]:
"""In-memory view for the introspection endpoint."""
active: Final[dict[str, int]] = {} # mutable-ok: counting accumulator for the snapshot
for binding in self._bindings.values():
active[binding.model] = active.get(binding.model, 0) + 1
return {
"router_name": self.router_name,
"available_models": list(self.models),
"decision_maker": self.decision_maker.snapshot(),
"verifier": type(self.verifier).__name__,
"active_programs": len(self._bindings),
"active_per_model": active,
"programs_bound": self.programs_bound,
"programs_completed": self.programs_completed,
"programs_evicted": self.programs_evicted,
"stateless_requests": self.stateless_requests,
"pending_verifications": self.pending_verifications,
"verifications_failed": self.verifications_failed,
"verifications_rejected": self.verifications_rejected,
"last_failure": self.last_failure,
"recent_accuracy": self._accuracy_by_model(),
"recent_feedback": [
{
"program_id": record.program_id,
"model": record.model,
"request_type": record.request_type,
"score": record.score,
"cost": record.cost,
}
for record in tuple(self._history)[-20:]
],
}
def request_type_of(prompt: str) -> RequestType:
return classify_prompt(prompt)

View file

@ -0,0 +1,235 @@
"""Verifiers score a finished program in [0, 1], off the critical path.
The contract is one coroutine. ORACLE ships three: any configured LiteLLM guardrail applied to the
program's final turn with its transcript as context (``llm_as_a_judge``, ``custom_code`` or a hosted
check), the score the client reported, and an import path for an operator's own class.
"""
import math
from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final, Protocol, runtime_checkable
from pydantic import TypeAdapter, ValidationError
import litellm
from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, REDACTED_BY_LITELLM
from litellm.exceptions import GuardrailRaisedException
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import OracleVerifierConfig
from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse
from .decision import load_object
_EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
_STR_MAPPING: Final = TypeAdapter(Mapping[str, object])
_STATUS_RECORDS: Final = TypeAdapter(Sequence[Mapping[str, object]])
_STATUS_RECORD_KEY: Final[str] = "standard_logging_guardrail_information"
_PROXY_OWNED_METADATA_KEYS: Final[frozenset[str]] = frozenset(
{_STATUS_RECORD_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "headers", "routing_decision"}
)
_BLOCKED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_intervened", "guardrail_flagged"})
_UNVERIFIED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_failed_to_respond", "not_run"})
@dataclass(frozen=True, slots=True)
class ProgramOutcome:
"""Everything a verifier may look at once a program has finished."""
program_id: str
model: str
prompt: str
response_text: str
cost: float
messages: Sequence[AllMessageValues] = ()
score: float | None = None
payload: Mapping[str, object] = field(default_factory=lambda: _EMPTY)
message_logging_off: bool = False
"""Whether any request of the program opted out of message logging; the router verifies under that opt-out."""
@runtime_checkable
class Verifier(Protocol):
async def verify(self, outcome: ProgramOutcome) -> float: ...
def clamp_score(value: float) -> float:
"""Clamp a score to [0, 1]. A NaN/inf score raises: it would corrupt a bandit cell for good."""
score: Final = float(value)
if not math.isfinite(score):
raise ValueError(f"score must be finite, got {score!r}")
return min(max(score, 0.0), 1.0)
def response_text(response: object) -> str:
"""The assistant text of a chat completion, or an empty string for anything else."""
if not isinstance(response, ModelResponse) or not response.choices:
return ""
content: Final = response.choices[0].message.content
return content if isinstance(content, str) else ""
class ReportedVerifier:
"""Trust the score the client reported with the program's last request or through the feedback endpoint."""
def __init__(self, default: float = 0.0) -> None:
self._default: Final[float] = default
async def verify(self, outcome: ProgramOutcome) -> float:
return clamp_score(outcome.score) if outcome.score is not None else self._default
def find_guardrail(name: str) -> CustomGuardrail | None:
"""The initialized guardrail registered under ``name``, if any."""
callbacks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(callback_type=CustomGuardrail)
guardrails: Final = tuple(callback for callback in callbacks if isinstance(callback, CustomGuardrail))
return next((guardrail for guardrail in guardrails if guardrail.guardrail_name == name), None)
def _redacted(outcome: ProgramOutcome) -> bool:
"""Whether LiteLLM replaced the program's final answer or transcript with its redaction placeholder."""
return outcome.response_text == REDACTED_BY_LITELLM or any(
_STR_MAPPING.validate_python(message).get("content") == REDACTED_BY_LITELLM for message in outcome.messages
)
def _verifier_metadata(outcome: ProgramOutcome) -> dict[str, object]: # mutable-ok: the guardrail writes into it
"""The feedback payload as the guardrail's request metadata.
The verdict is read from the status record the guardrail writes here, so the caller's payload must not
pre-seed that key (a non-record entry would make the whole list unreadable and hide a rejection), it
cannot rename the program, and it cannot pose as the proxy: the ``user_api_key_*`` identity a guardrail
may forward to its vendor, the sub-call origin and the request headers are the proxy's to stamp.
"""
return {
**{key: value for key, value in outcome.payload.items() if not _proxy_owned(key)},
"program_id": outcome.program_id,
}
def _proxy_owned(key: str) -> bool:
return key.startswith("user_api_key") or key in _PROXY_OWNED_METADATA_KEYS
def _status_records(bucket: object) -> Sequence[Mapping[str, object]]:
"""The ``standard_logging_guardrail_information`` records of one metadata bucket, or none."""
try:
return _STATUS_RECORDS.validate_python(_STR_MAPPING.validate_python(bucket).get(_STATUS_RECORD_KEY) or ())
except ValidationError:
return ()
def recorded_status(request_data: Mapping[str, object], guardrail_name: str) -> str | None:
"""The last status ``guardrail_name`` recorded for itself in ``request_data`` while it ran, if any.
Every first-party guardrail writes a ``standard_logging_guardrail_information`` record into the
request's metadata bucket (``success``, ``guardrail_intervened``, ``guardrail_failed_to_respond``,
...), which is the only way to tell a judge that failed the answer from one that failed to answer.
"""
records: Final = (
*_status_records(request_data.get("metadata")),
*_status_records(request_data.get("litellm_metadata")),
)
statuses: Final = tuple(
str(record["guardrail_status"])
for record in records
if record.get("guardrail_name") in (guardrail_name, None) and record.get("guardrail_status")
)
return statuses[-1] if statuses else None
async def _apply(
guardrail: CustomGuardrail,
inputs: GenericGuardrailAPIInputs,
request_data: dict[str, object], # mutable-ok: the guardrail writes its status into it
) -> Exception | None:
"""Run the guardrail; the exception it raised to signal its verdict, or ``None`` when it returned."""
try:
await guardrail.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # upstream types request_data as a bare dict
inputs=inputs, request_data=request_data, input_type="response"
)
except Exception as error: # noqa: BLE001 # a guardrail may signal its verdict by raising; the caller classifies it
return error
return None
def _is_http_block(error: Exception) -> bool:
"""A guardrail without a status record that blocks by raising FastAPI's ``HTTPException``."""
try:
from fastapi import HTTPException
except ImportError: # the SDK without the proxy extra: nothing can raise it
return False
return isinstance(error, HTTPException)
class GuardrailVerifier:
"""Apply a configured LiteLLM guardrail to the program's final turn: it passes or it is blocked.
The guardrail sees the final assistant text as ``texts``, the program's transcript as
``structured_messages`` and the feedback payload as ``request_data["metadata"]``, so an
``llm_as_a_judge`` guardrail judges the whole conversation and a ``custom_code`` guardrail can read
whatever the harness posted with the feedback.
The verdict is read from the status the guardrail records for itself (so a judge configured with
``on_failure: log`` still scores 0 when the answer fails), and falls back to how it raised: a
``GuardrailRaisedException`` with ``blocked_content`` or an ``HTTPException`` is a block. A
guardrail that could not evaluate the program (``guardrail_failed_to_respond``, an unreachable
backend) raises, so the program counts as a failed verification and teaches the decision maker
nothing instead of passing it.
"""
def __init__(self, guardrail_name: str, lookup: Callable[[str], CustomGuardrail | None] = find_guardrail) -> None:
self._name: Final[str] = guardrail_name
self._lookup: Final = lookup
async def verify(self, outcome: ProgramOutcome) -> float:
guardrail: Final = self._lookup(self._name)
if guardrail is None:
raise LookupError(f"guardrail {self._name!r} is not initialized on this proxy")
if not outcome.response_text.strip():
# A judge skips an empty answer without recording a verdict; that must not read as a pass.
raise RuntimeError(f"guardrail {self._name!r}: the program has no response text to verify")
if _redacted(outcome):
# The request opted out of message logging, so the router only ever saw LiteLLM's placeholder.
raise RuntimeError(f"guardrail {self._name!r}: the program's final turn was redacted before verification")
transcript: Final = list(outcome.messages) or [{"role": "user", "content": outcome.prompt}]
inputs: Final = GenericGuardrailAPIInputs(
texts=[outcome.response_text], model=outcome.model, structured_messages=transcript
)
request_data: Final[dict[str, object]] = { # mutable-ok: the guardrail records its status into it
"model": outcome.model,
"metadata": _verifier_metadata(outcome),
}
verdict_error: Final = await _apply(guardrail, inputs, request_data)
status: Final = recorded_status(request_data, self._name)
if status in _UNVERIFIED_STATUSES:
raise RuntimeError(f"guardrail {self._name!r} did not evaluate program {outcome.program_id!r}: {status}")
if status in _BLOCKED_STATUSES:
return 0.0
if status == "success" or verdict_error is None:
return 1.0
if isinstance(verdict_error, GuardrailRaisedException):
if verdict_error.blocked_content:
return 0.0
raise verdict_error # the guardrail could not reach a verdict
if _is_http_block(verdict_error):
return 0.0
raise verdict_error
def build_verifier(config: OracleVerifierConfig) -> Verifier:
match config.type:
case "reported":
return ReportedVerifier(default=config.default_score)
case "guardrail":
return GuardrailVerifier(config.guardrail or "")
case "custom":
factory: Final = load_object(config.path or "")
if not callable(factory):
raise TypeError(f"oracle_router verifier.path {config.path!r} is not callable")
instance: Final = factory()
if not isinstance(instance, Verifier):
raise TypeError(f"{config.path!r} must build an object with an async verify(outcome) method")
return instance

View file

@ -0,0 +1,31 @@
"""Types for the ORACLE router management endpoints."""
from collections.abc import Mapping, Sequence
from pydantic import BaseModel, Field
class OracleRouterFeedbackRequest(BaseModel, frozen=True):
"""Report that a program finished. ``score`` is optional when the router's verifier grades it."""
program_id: str = Field(min_length=1)
score: float | None = Field(default=None, ge=0.0, le=1.0)
cost: float | None = Field(
default=None, ge=0.0, description="Overrides the spend LiteLLM accumulated for the program"
)
payload: Mapping[str, object] = Field(default_factory=dict, description="Passed to the verifier unchanged")
class OracleRouterFeedbackResponse(BaseModel, frozen=True):
program_id: str
router_name: str
model: str
pending_verifications: int
deferred: bool = Field(
default=False,
description="No response of the program has been observed yet; its next one completes it with this feedback",
)
class OracleRouterStateResponse(BaseModel, frozen=True):
routers: Sequence[Mapping[str, object]]

View file

@ -675,6 +675,8 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
# adaptive-router params
adaptive_router_default_model: str | None = None
adaptive_router_config: dict | None = None
# oracle-router params
oracle_router_config: Mapping[str, object] | None = None
# quality-router params
quality_router_config: dict | None = None
quality_router_default_model: str | None = None
@ -1531,6 +1533,82 @@ class AdaptiveRouterPreferences(LiteLLMBaseModel):
strengths: list[RequestType] = Field(default_factory=list)
OracleDecisionMakerType: TypeAlias = Literal["thompson", "fixed", "pre_routing", "custom"]
OracleVerifierType: TypeAlias = Literal["reported", "guardrail", "custom"]
class OracleDecisionMakerConfig(BaseModel, frozen=True):
"""Which policy binds a program to a model, and its knobs.
``thompson`` (default) is the adaptive router's bandit trained on verified program outcomes; ``weights``
are that bandit's quality/cost weights. ``fixed`` always picks ``model``. ``pre_routing`` lets the strategy
router named by ``router`` (a complexity, quality, adaptive or semantic router in the same model_list)
propose the model once per program. ``custom`` imports ``path`` (``package.module:Factory``) and calls it
with the model tuple; it is honoured only for deployments in the proxy config file, never for models
written through the API.
"""
type: OracleDecisionMakerType = "thompson"
weights: AdaptiveRouterWeights = Field(default_factory=AdaptiveRouterWeights)
seed: int = 0
model: str | None = None
router: str | None = None
path: str | None = None
@model_validator(mode="after")
def _required_fields(self) -> "OracleDecisionMakerConfig":
required: Final = {"fixed": self.model, "pre_routing": self.router, "custom": self.path}
if self.type in required and not required[self.type]:
names: Final = {"fixed": "model", "pre_routing": "router", "custom": "path"}
raise ValueError(f"decision_maker.type={self.type!r} requires decision_maker.{names[self.type]}")
return self
class OracleVerifierConfig(BaseModel, frozen=True):
"""How a finished program is scored.
``guardrail`` applies the configured LiteLLM guardrail ``guardrail`` (for example an ``llm_as_a_judge``
guardrail) to the program's final turn with the transcript as context. ``reported`` trusts
``metadata.program_score`` or the feedback endpoint. ``custom`` imports ``path`` and calls it with no
arguments; it is honoured only for deployments in the proxy config file, never for models written
through the API.
"""
type: OracleVerifierType = "reported"
default_score: float = Field(default=0.0, ge=0.0, le=1.0)
guardrail: str | None = None
path: str | None = None
@model_validator(mode="after")
def _required_fields(self) -> "OracleVerifierConfig":
required: Final = {"guardrail": self.guardrail, "custom": self.path}
if self.type in required and not required[self.type]:
names: Final = {"guardrail": "guardrail", "custom": "path"}
raise ValueError(f"verifier.type={self.type!r} requires verifier.{names[self.type]}")
return self
class OracleRouterConfig(BaseModel, frozen=True):
"""``litellm_params.oracle_router_config`` of an ``auto_router/oracle_router`` deployment."""
available_models: tuple[str, ...] = Field(min_length=1)
decision_maker: OracleDecisionMakerConfig = Field(default_factory=OracleDecisionMakerConfig)
verifier: OracleVerifierConfig = Field(default_factory=OracleVerifierConfig)
program_id_key: str = "program_id"
program_ttl_seconds: int = Field(default=24 * 3600, ge=1)
max_programs: int = Field(default=10_000, ge=1)
max_pending_verifications: int = Field(default=64, ge=1)
verification_timeout_seconds: float = Field(default=120.0, gt=0)
@model_validator(mode="after")
def _fixed_model_is_available(self) -> "OracleRouterConfig":
if len(self.available_models) != len(frozenset(self.available_models)):
raise ValueError("available_models must be distinct")
if self.decision_maker.type == "fixed" and self.decision_maker.model not in self.available_models:
raise ValueError("decision_maker.model must be one of available_models")
return self
def reject_server_owned_wif_params(body: Mapping[str, object]) -> None:
"""Raise ``ValueError`` if a mapping that did not come from deployment config carries a
server-owned workload identity federation or OAuth token-exchange field, or an ``api_key``

View file

@ -3373,7 +3373,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
"""Per-request provenance for a pre-routing strategy (auto-router) decision."""
router_model_name: str
router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]]
router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic", "oracle"]]
router_config_id: ReadOnly[str]
router_config_updated_at: ReadOnly[str]
router_config_fingerprint: ReadOnly[str]

View file

@ -1085,7 +1085,7 @@ class TestClearCache:
Test that clear_cache successfully clears router model caches and reloads models.
"""
mock_router = MagicMock()
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = ["openai/gpt-4o", "openai/gpt-4o-mini"]
@ -1125,7 +1125,7 @@ class TestClearCache:
# entries are auto_router/* deployments (so their router-map entries should be
# cleared for reload); the config-defined router is preserved.
mock_router = MagicMock()
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@ -1209,7 +1209,7 @@ class TestClearCache:
)
mock_router = MagicMock()
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@ -1270,7 +1270,7 @@ class TestClearCachePreservesConfigRouters:
)
mock_router = MagicMock()
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@ -1315,7 +1315,7 @@ class TestClearCachePreservesConfigRouters:
from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache
mock_router = MagicMock()
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@ -1354,7 +1354,7 @@ class TestClearCachePreservesConfigRouters:
from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache
mock_router = MagicMock()
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.registry.return_value.all.return_value = [
{
@ -1367,11 +1367,17 @@ class TestClearCachePreservesConfigRouters:
"model_info": {"id": "db-a", "db_model": True},
"litellm_params": {"model": "auto_router/adaptive_router/a1"},
},
{
"model_name": "o1",
"model_info": {"id": "db-o", "db_model": True},
"litellm_params": {"model": "auto_router/oracle_router/o1"},
},
]
plugins["auto"] = {}
plugins["complexity"] = {}
plugins["quality"] = {"q1": MagicMock()}
plugins["adaptive"] = {"a1": MagicMock()}
plugins["oracle"] = {"o1": MagicMock()}
mock_config = MagicMock()
mock_config.add_deployment_locked = AsyncMock(
@ -1389,6 +1395,7 @@ class TestClearCachePreservesConfigRouters:
assert "q1" not in plugins["quality"]
assert "a1" not in plugins["adaptive"]
assert "o1" not in plugins["oracle"]
class TestDeleteModelClearsRouterRegistry:
@ -1503,7 +1510,7 @@ class TestDeleteModelClearsRouterRegistry:
mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row)
mock_router = MagicMock()
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}}
plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}}
mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__
mock_router.apply.return_value = ApplyResult(
version=1,

View file

@ -0,0 +1,190 @@
"""POST /oracle_router/feedback and GET /oracle_router/state."""
import asyncio
from unittest.mock import patch
import pytest
from fastapi import HTTPException
from pydantic import ValidationError
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.management_endpoints.oracle_router_endpoints import (
get_oracle_router_state,
submit_oracle_router_feedback,
)
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
from litellm.router_strategy.oracle_router.verifier import ReportedVerifier
from litellm.types.management_endpoints.oracle_router_endpoints import OracleRouterFeedbackRequest
from litellm.types.router import OracleRouterConfig, TaggedPreRoutingStrategy
ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="admin-hash")
OWNER = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="owner-hash")
STRANGER = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="other-hash")
class _Maker:
def __init__(self) -> None:
self.updates = []
@property
def models(self):
return ("smart", "fast")
async def select(self, context):
return "smart"
def update(self, context, model, score):
self.updates.append((model, score))
def snapshot(self):
return {"type": "test"}
class _Registry:
def __init__(self, oracle: OracleRouter) -> None:
self._oracle_routers = {oracle.router_name: [TaggedPreRoutingStrategy(tags=(), strategy=oracle)]}
def plugins(self, kind: str):
assert kind == "oracle"
return self._oracle_routers
class _LLMRouter:
def __init__(self, oracle: OracleRouter) -> None:
self._registry = _Registry(oracle)
def registry(self) -> _Registry:
return self._registry
@pytest.fixture
def oracle():
maker = _Maker()
router = OracleRouter("oracle", OracleRouterConfig(available_models=["smart", "fast"]), maker, ReportedVerifier())
router.maker = maker
return router
async def _bind(oracle: OracleRouter, program_id: str, api_key_hash: str | None = "owner-hash"):
metadata = {"program_id": program_id, **({"user_api_key_hash": api_key_hash} if api_key_hash else {})}
await oracle.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": metadata}, messages=[{"role": "user", "content": "t"}]
)
@pytest.mark.asyncio
async def test_feedback_completes_the_program_and_the_router_learns(oracle):
await _bind(oracle, "task-1")
oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash")
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
response = await submit_oracle_router_feedback(
OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER
)
assert response.model_dump() == {
"program_id": "task-1",
"router_name": "oracle",
"model": "smart",
"pending_verifications": 1,
"deferred": False,
}
assert oracle.binding("task-1", "owner-hash") is None
await oracle.drain()
assert oracle.maker.updates == [("smart", 1.0)]
@pytest.mark.asyncio
async def test_feedback_is_rejected_with_503_while_the_verification_backlog_is_full():
class _Stuck:
async def verify(self, outcome):
await asyncio.sleep(5.0)
return 1.0
config = OracleRouterConfig(available_models=["smart", "fast"], max_pending_verifications=1)
oracle = OracleRouter("oracle", config, _Maker(), _Stuck())
for program_id in ("task-1", "task-2"):
await _bind(oracle, program_id)
oracle.observe_request(program_id, cost=0.2, response_text="done", owner="owner-hash")
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER)
with pytest.raises(HTTPException) as rejected:
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-2", score=1.0), OWNER)
assert rejected.value.status_code == 503 and "max_pending_verifications=1" in rejected.value.detail["error"]
assert oracle.binding("task-2", "owner-hash") is not None
for task in tuple(oracle._tasks):
task.cancel()
@pytest.mark.asyncio
async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it(oracle):
await _bind(oracle, "task-1")
oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash")
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
with pytest.raises(HTTPException) as unseen:
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), STRANGER)
assert unseen.value.status_code == 404 and oracle.binding("task-1", "owner-hash") is not None
response = await submit_oracle_router_feedback(
OracleRouterFeedbackRequest(program_id="task-1", score=0.0), ADMIN
)
assert response.program_id == "task-1" and oracle.binding("task-1", "owner-hash") is None
await oracle.drain()
@pytest.mark.asyncio
async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across_keys(oracle):
await _bind(oracle, "task-1", "owner-hash")
await _bind(oracle, "task-1", "other-hash")
oracle.observe_request("task-1", cost=0.2, response_text="done", owner="other-hash")
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
response = await submit_oracle_router_feedback(
OracleRouterFeedbackRequest(program_id="task-1", score=0.5), STRANGER
)
assert response.program_id == "task-1"
assert oracle.binding("task-1", "other-hash") is None and oracle.binding("task-1", "owner-hash") is not None
await oracle.drain()
@pytest.mark.asyncio
async def test_feedback_before_any_response_is_deferred_until_one_is_observed(oracle):
await _bind(oracle, "task-1") # bound, but its first response has not been observed yet
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
response = await submit_oracle_router_feedback(
OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER
)
assert response.deferred is True and response.pending_verifications == 0
assert oracle.binding("task-1", "owner-hash") is not None and oracle.maker.updates == []
oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash")
assert oracle.binding("task-1", "owner-hash") is None
await oracle.drain()
assert oracle.maker.updates == [("smart", 1.0)]
@pytest.mark.asyncio
async def test_feedback_for_an_unknown_program_or_without_routers_is_404(oracle):
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
with pytest.raises(HTTPException) as unknown:
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="nope"), OWNER)
assert unknown.value.status_code == 404
with patch("litellm.proxy.proxy_server.llm_router", None):
with pytest.raises(HTTPException) as none:
await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="nope"), OWNER)
assert none.value.status_code == 404
@pytest.mark.asyncio
async def test_state_is_admin_only_and_lists_every_router(oracle):
await _bind(oracle, "task-1")
with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
with pytest.raises(HTTPException) as denied:
await get_oracle_router_state(OWNER)
assert denied.value.status_code == 403
state = await get_oracle_router_state(ADMIN)
assert len(state.routers) == 1
assert state.routers[0]["router_name"] == "oracle" and state.routers[0]["active_programs"] == 1
assert state.routers[0]["decision_maker"] == {"type": "test"}
def test_feedback_request_validates_score_range():
with pytest.raises(ValidationError, match="score"):
OracleRouterFeedbackRequest(program_id="p", score=1.5)
with pytest.raises(ValidationError, match="program_id"):
OracleRouterFeedbackRequest(program_id="")

View file

@ -0,0 +1,210 @@
"""Decision makers: each learns or proposes the way its contract says."""
import pytest
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
initialize_standard_callback_dynamic_params,
)
from litellm.litellm_core_utils.internal_call_metadata import effective_turn_off_message_logging
from litellm.router_strategy.oracle_router.decision import (
DecisionMaker,
FixedDecisionMaker,
PreRoutingDecisionMaker,
ProgramContext,
ThompsonDecisionMaker,
build_decision_maker,
load_object,
)
from litellm.types.router import AdaptiveRouterWeights, OracleDecisionMakerConfig, PreRoutingHookResponse, RequestType
MODELS = ("smart", "fast")
COSTS = {"smart": 1e-5, "fast": 1e-6}
def _ctx(prompt: str = "fix the failing test", request_type: RequestType = RequestType.GENERAL) -> ProgramContext:
return ProgramContext(program_id="p", prompt=prompt, request_type=request_type)
@pytest.mark.asyncio
async def test_thompson_learns_a_different_winner_per_request_type():
maker = ThompsonDecisionMaker(MODELS, COSTS, weights=AdaptiveRouterWeights(quality=1.0, cost=0.0), seed=0)
for _ in range(80):
for request_type, good in ((RequestType.CODE_GENERATION, "smart"), (RequestType.WRITING, "fast")):
ctx = _ctx(request_type=request_type)
chosen = await maker.select(ctx)
maker.update(ctx, chosen, 1.0 if chosen == good else 0.0)
code, writing = _ctx(request_type=RequestType.CODE_GENERATION), _ctx(request_type=RequestType.WRITING)
assert maker.estimate(code, "smart") > maker.estimate(code, "fast")
assert maker.estimate(writing, "fast") > maker.estimate(writing, "smart")
picks = [await maker.select(code) for _ in range(30)]
assert picks.count("smart") > 24
assert maker.snapshot()["cells"]["code_generation/smart"]["samples"] > 0
@pytest.mark.asyncio
async def test_thompson_cost_weight_prefers_the_cheap_model_when_quality_ties():
maker = ThompsonDecisionMaker(MODELS, COSTS, weights=AdaptiveRouterWeights(quality=0.3, cost=0.7), seed=1)
ctx = _ctx()
for _ in range(40):
maker.update(ctx, "smart", 0.8)
maker.update(ctx, "fast", 0.8)
picks = [await maker.select(ctx) for _ in range(30)]
assert picks.count("fast") > 24
@pytest.mark.asyncio
async def test_thompson_priors_come_from_adaptive_router_preferences():
from litellm.types.router import AdaptiveRouterPreferences
prefs = {
"smart": AdaptiveRouterPreferences(quality_tier=3, strengths=[]),
"fast": AdaptiveRouterPreferences(quality_tier=1, strengths=[]),
}
maker = ThompsonDecisionMaker(
MODELS, COSTS, weights=AdaptiveRouterWeights(quality=1.0, cost=0.0), model_prefs=prefs, seed=0
)
ctx = _ctx()
assert maker.estimate(ctx, "smart") > maker.estimate(ctx, "fast")
picks = [await maker.select(ctx) for _ in range(30)]
assert picks.count("smart") > 24
def test_thompson_rejects_unknown_model_and_empty_pool():
maker = ThompsonDecisionMaker(MODELS, COSTS)
with pytest.raises(KeyError):
maker.update(_ctx(), "other", 1.0)
with pytest.raises(ValueError, match="at least one model"):
ThompsonDecisionMaker((), COSTS)
@pytest.mark.asyncio
async def test_fixed_always_returns_its_model_and_counts_updates():
maker = FixedDecisionMaker(MODELS, model="fast")
assert await maker.select(_ctx()) == "fast"
maker.update(_ctx(), "fast", 1.0)
assert maker.snapshot() == {"type": "fixed", "model": "fast", "updates": 1}
with pytest.raises(KeyError):
FixedDecisionMaker(MODELS, model="nope")
class _FakeStrategy:
def __init__(self, answer: str | None) -> None:
self.answer = answer
self.calls: list[dict] = []
async def async_pre_routing_hook(self, model, request_kwargs, messages=None, input=None, specific_deployment=False):
self.calls.append({"model": model, "messages": messages})
self.request_kwargs = request_kwargs
self.privacy = (
effective_turn_off_message_logging(request_kwargs),
initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging"),
)
return None if self.answer is None else PreRoutingHookResponse(model=self.answer, messages=messages)
@pytest.mark.asyncio
async def test_pre_routing_uses_the_wrapped_strategy_proposal_once_per_program():
strategy = _FakeStrategy("fast")
maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy)
assert await maker.select(_ctx("write a haiku")) == "fast"
assert strategy.calls == [{"model": "tiered", "messages": [{"role": "user", "content": "write a haiku"}]}]
assert maker.snapshot()["proposals"] == {"fast": 1}
@pytest.mark.asyncio
async def test_pre_routing_forwards_the_program_s_message_logging_opt_out_to_the_wrapped_strategy():
strategy = _FakeStrategy("fast")
maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy)
private = ProgramContext(
program_id="p", prompt="secret", request_type=RequestType.GENERAL, message_logging_off=True
)
assert await maker.select(private) == "fast"
assert strategy.privacy == (True, True)
assert await maker.select(_ctx("public")) == "fast"
assert strategy.privacy == (None, None)
@pytest.mark.asyncio
async def test_pre_routing_forwards_the_caller_s_identity_so_a_classifier_call_is_billed_to_the_caller():
strategy = _FakeStrategy("fast")
maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy)
caller = {"user_api_key_hash": "hash-1", "user_api_key_team_id": "team-1", "user_api_key_user_id": "user-1"}
private = ProgramContext(
program_id="p", prompt="task", request_type=RequestType.GENERAL, message_logging_off=True, caller=caller
)
assert await maker.select(private) == "fast"
assert strategy.request_kwargs == {"metadata": {**caller, "turn_off_message_logging": True}}
first_call = strategy.request_kwargs
assert await maker.select(private) == "fast"
assert strategy.request_kwargs == first_call and strategy.request_kwargs is not first_call
assert await maker.select(_ctx()) == "fast"
assert strategy.request_kwargs == {"metadata": {}}
@pytest.mark.asyncio
async def test_pre_routing_falls_back_to_the_first_model_when_the_proposal_is_unusable():
assert await PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy(None)).select(_ctx()) == "smart"
maker = PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy("not-configured"))
assert await maker.select(_ctx()) == "smart"
assert maker.snapshot()["fallbacks"] == 1
class CustomMaker:
def __init__(self, models: tuple[str, ...]) -> None:
self._models = models
@property
def models(self) -> tuple[str, ...]:
return self._models
async def select(self, context: ProgramContext) -> str:
return self._models[-1]
def update(self, context: ProgramContext, model: str, score: float) -> None:
pass
def snapshot(self):
return {"type": "custom"}
class NotAMaker:
def __init__(self, models: tuple[str, ...]) -> None:
pass
@pytest.mark.asyncio
async def test_build_decision_maker_by_type():
resolve = {"tiered": _FakeStrategy("smart")}.get
thompson = build_decision_maker(OracleDecisionMakerConfig(), MODELS, COSTS, resolve)
assert isinstance(thompson, ThompsonDecisionMaker) and await thompson.select(_ctx()) in MODELS
assert isinstance(
build_decision_maker(OracleDecisionMakerConfig(type="fixed", model="fast"), MODELS, COSTS, resolve),
FixedDecisionMaker,
)
pre = build_decision_maker(OracleDecisionMakerConfig(type="pre_routing", router="tiered"), MODELS, COSTS, resolve)
assert isinstance(pre, PreRoutingDecisionMaker) and await pre.select(_ctx()) == "smart"
with pytest.raises(ValueError, match="not a configured strategy router"):
build_decision_maker(OracleDecisionMakerConfig(type="pre_routing", router="missing"), MODELS, COSTS, resolve)
custom = build_decision_maker(
OracleDecisionMakerConfig(type="custom", path=f"{__name__}:CustomMaker"), MODELS, COSTS, resolve
)
assert isinstance(custom, DecisionMaker) and await custom.select(_ctx()) == "fast"
with pytest.raises(TypeError):
build_decision_maker(
OracleDecisionMakerConfig(type="custom", path=f"{__name__}:NotAMaker"), MODELS, COSTS, resolve
)
def test_load_object_requires_module_colon_attribute():
with pytest.raises(ValueError, match=r"package\.module:Attribute"):
load_object("no_colon_here")
assert load_object(f"{__name__}:CustomMaker") is CustomMaker
def test_config_requires_the_fields_each_type_reads():
with pytest.raises(ValueError, match=r"decision_maker\.model"):
OracleDecisionMakerConfig(type="fixed")
with pytest.raises(ValueError, match=r"decision_maker\.router"):
OracleDecisionMakerConfig(type="pre_routing")
with pytest.raises(ValueError, match=r"decision_maker\.path"):
OracleDecisionMakerConfig(type="custom")

View file

@ -0,0 +1,200 @@
"""The post-call hook feeds spend and completion signals from LiteLLM's logging payload into the router."""
import asyncio
import pytest
from litellm.router_strategy.oracle_router.config import (
CHOSEN_MODEL_METADATA_KEY,
PROGRAM_ID_METADATA_KEY,
RESPONSE_HEADER,
)
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
from litellm.router_strategy.oracle_router.oracle_router import OracleRouter
from litellm.router_strategy.oracle_router.verifier import ReportedVerifier
from litellm.types.router import OracleRouterConfig
from litellm.types.utils import Choices, Message, ModelResponse
class _Maker:
def __init__(self) -> None:
self.updates = []
@property
def models(self):
return ("smart", "fast")
async def select(self, context):
return "smart"
def update(self, context, model, score):
self.updates.append((model, score))
def snapshot(self):
return {}
ROUTED = {"router_model_name": "oracle", "router_type": "oracle"} # what the Router stamps for this router
def _kwargs(program_id="t1", cost=0.01, routing_decision=ROUTED, **metadata):
stamped = {PROGRAM_ID_METADATA_KEY: program_id, **metadata}
if routing_decision is not None:
stamped["routing_decision"] = routing_decision
return {
"model": "openai/gpt-4o",
"messages": [{"role": "user", "content": "task"}],
"litellm_params": {"metadata": stamped},
"response_cost": cost,
}
def _response(text="answer") -> ModelResponse:
return ModelResponse(choices=[Choices(message=Message(content=text, role="assistant"))])
@pytest.fixture
def bound():
maker = _Maker()
router = OracleRouter("oracle", OracleRouterConfig(available_models=["smart", "fast"]), maker, ReportedVerifier())
asyncio.run(
router.async_pre_routing_hook(
model="oracle",
request_kwargs={"metadata": {"program_id": "t1"}},
messages=[{"role": "user", "content": "task"}],
)
)
return router, maker, OracleRouterPostCallHook(router)
@pytest.mark.asyncio
async def test_success_events_accumulate_spend_and_the_latest_answer(bound):
router, _, hook = bound
await hook.async_log_success_event(_kwargs(cost=0.01), _response("first"), None, None)
await hook.async_log_success_event(_kwargs(cost=0.02), _response("second"), None, None)
binding = router.binding("t1")
assert binding.requests == 2 and binding.cost == pytest.approx(0.03) and binding.last_response_text == "second"
assert list(binding.last_messages) == [{"role": "user", "content": "task"}]
@pytest.mark.asyncio
async def test_a_turn_that_opted_out_of_message_logging_marks_the_program_private_for_good(bound):
router, _, hook = bound
await hook.async_log_success_event(_kwargs(cost=0.01), _response("first"), None, None)
assert router.binding("t1").message_logging_off is False
await hook.async_log_success_event(
_kwargs(cost=0.01, turn_off_message_logging=True), _response("second"), None, None
)
assert router.binding("t1").message_logging_off is True
await hook.async_log_success_event(_kwargs(cost=0.01), _response("third"), None, None)
assert router.binding("t1").message_logging_off is True
@pytest.mark.asyncio
async def test_a_private_turn_that_failed_still_marks_the_program_private(bound):
router, _, hook = bound
await hook.async_log_failure_event(_kwargs(cost=0.5, turn_off_message_logging=True), None, None, None)
assert router.binding("t1").message_logging_off is True and router.binding("t1").requests == 0
@pytest.mark.asyncio
async def test_a_turn_litellm_resolved_as_private_marks_the_program_private(bound):
router, _, hook = bound
resolved = {**_kwargs(cost=0.01), "standard_callback_dynamic_params": {"turn_off_message_logging": True}}
await hook.async_log_success_event(resolved, _response("first"), None, None)
assert router.binding("t1").message_logging_off is True
@pytest.mark.asyncio
async def test_program_done_completes_the_program_with_the_reported_score(bound):
router, maker, hook = bound
await hook.async_log_success_event(
_kwargs(cost=0.05, program_done=True, program_score="1"), _response(), None, None
)
assert router.binding("t1") is None
await router.drain()
assert maker.updates == [("smart", 1.0)]
@pytest.mark.asyncio
async def test_a_non_finite_program_score_is_treated_as_no_score(bound):
router, maker, hook = bound
await hook.async_log_success_event(
_kwargs(cost=0.05, program_done=True, program_score="nan"), _response(), None, None
)
assert router.binding("t1") is None
await router.drain()
assert maker.updates == [("smart", 0.0)] # the reported verifier's default, never NaN
@pytest.mark.asyncio
async def test_failure_event_adds_no_spend_and_program_done_on_it_waits_for_a_response(bound):
router, maker, hook = bound
await hook.async_log_failure_event(_kwargs(cost=0.5), None, None, None)
assert router.binding("t1").cost == 0.0
await hook.async_log_failure_event(_kwargs(program_done=True), None, None, None)
assert router.binding("t1") is not None and maker.updates == [] # no answer to verify yet: deferred
await hook.async_log_success_event(_kwargs(cost=0.1), _response("late answer"), None, None)
assert router.binding("t1") is None
await router.drain()
assert maker.updates == [("smart", 0.0)]
@pytest.mark.asyncio
async def test_events_for_unknown_or_untagged_programs_are_ignored(bound):
router, _, hook = bound
await hook.async_log_success_event(
{"litellm_params": {"metadata": {}}, "response_cost": 1.0}, _response(), None, None
)
await hook.async_log_success_event(_kwargs(program_id="other"), _response(), None, None)
assert router.binding("t1").requests == 0 and router.binding("other") is None
@pytest.mark.asyncio
async def test_events_from_requests_this_router_did_not_route_are_ignored(bound):
router, maker, hook = bound
unrouted = _kwargs(cost=0.5, program_done=True, program_score="1", routing_decision=None)
await hook.async_log_success_event(unrouted, _response(), None, None)
elsewhere = _kwargs(
cost=0.5, program_done=True, program_score="1", routing_decision={**ROUTED, "router_model_name": "x"}
)
await hook.async_log_success_event(elsewhere, _response(), None, None)
assert router.binding("t1").requests == 0 and maker.updates == []
@pytest.mark.asyncio
async def test_events_only_reach_the_program_of_the_key_that_sent_them(bound):
router, maker, hook = bound # t1 was started without a key
stranger = _kwargs(cost=0.5, program_done=True, program_score="1", user_api_key_hash="other-key")
await hook.async_log_success_event(stranger, _response(), None, None)
assert router.binding("t1").requests == 0 and maker.updates == []
await router.async_pre_routing_hook(
model="oracle",
request_kwargs={"metadata": {"program_id": "t1", "user_api_key_hash": "other-key"}},
messages=[{"role": "user", "content": "task"}],
)
await hook.async_log_success_event(stranger, _response(), None, None)
assert router.binding("t1", "other-key") is None and router.binding("t1").requests == 0
await router.drain()
assert maker.updates == [("smart", 1.0)]
@pytest.mark.asyncio
async def test_response_header_names_the_bound_model(bound):
_, _, hook = bound
headers = await hook.async_post_call_response_headers_hook(
{"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart", "routing_decision": ROUTED}}, None, None
)
assert headers == {RESPONSE_HEADER: "smart"}
assert (
await hook.async_post_call_response_headers_hook({"metadata": {"routing_decision": ROUTED}}, None, None) is None
)
forged = {"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart"}} # a request this router did not route
assert await hook.async_post_call_response_headers_hook(forged, None, None) is None
@pytest.mark.asyncio
async def test_response_header_reads_the_litellm_metadata_bucket_too(bound):
_, _, hook = bound
data = {"metadata": {}, "litellm_metadata": {CHOSEN_MODEL_METADATA_KEY: "fast", "routing_decision": ROUTED}}
assert await hook.async_post_call_response_headers_hook(data, None, None) == {RESPONSE_HEADER: "fast"}

View file

@ -0,0 +1,445 @@
"""OracleRouter: sticky program bindings, program id resolution, delayed feedback, eviction."""
import asyncio
import math
import pytest
from litellm.litellm_core_utils.core_helpers import budget_reservation_from_metadata
from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
initialize_standard_callback_dynamic_params,
)
from litellm.router_strategy.oracle_router.config import (
CHOSEN_MODEL_METADATA_KEY,
PROGRAM_ID_HEADER,
PROGRAM_ID_METADATA_KEY,
)
from litellm.router_strategy.oracle_router.decision import ProgramContext
from litellm.router_strategy.oracle_router.oracle_router import (
OracleRouter,
VerificationBacklogFull,
first_user_text,
resolve_program_id,
)
from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, ReportedVerifier
from litellm.types.router import OracleRouterConfig
MODELS = ("smart", "fast")
class _RecordingMaker:
"""Alternates models per bind and records every update in arrival order."""
def __init__(self) -> None:
self.selections = 0
self.updates: list[tuple[str, str, float]] = []
self.contexts: list[ProgramContext] = []
@property
def models(self):
return MODELS
async def select(self, context: ProgramContext) -> str:
self.selections += 1
self.contexts.append(context)
return MODELS[(self.selections - 1) % 2]
def update(self, context: ProgramContext, model: str, score: float) -> None:
self.updates.append((context.program_id, model, score))
def snapshot(self):
return {"type": "recording"}
class _SlowVerifier:
def __init__(self, delays: dict[str, float]) -> None:
self.delays = delays
async def verify(self, outcome: ProgramOutcome) -> float:
await asyncio.sleep(self.delays.get(outcome.program_id, 0.0))
return outcome.score if outcome.score is not None else 0.0
def _router(verifier=None, maker=None, clock=None, **config) -> tuple[OracleRouter, _RecordingMaker]:
maker = maker or _RecordingMaker()
router = OracleRouter(
router_name="oracle",
config=OracleRouterConfig(available_models=list(MODELS), **config),
decision_maker=maker,
verifier=verifier or ReportedVerifier(),
**({"clock": clock} if clock else {}),
)
return router, maker
def _messages(text: str = "fix the failing test in src/parser.py"):
return [{"role": "system", "content": "be terse"}, {"role": "user", "content": text}]
def test_resolve_program_id_precedence_and_header():
assert resolve_program_id({"metadata": {"program_id": "a", "litellm_session_id": "b"}}, "program_id") == "a"
assert resolve_program_id({"metadata": {"litellm_session_id": "b"}}, "program_id") == "b"
assert resolve_program_id({"metadata": {"session_id": 7}}, "program_id") == "7"
assert resolve_program_id({"litellm_session_id": "lp"}, "program_id") == "lp"
assert resolve_program_id({"headers": {PROGRAM_ID_HEADER.upper(): "h"}}, "program_id") == "h"
assert resolve_program_id({"metadata": {"task": "t"}}, "task") == "t"
assert resolve_program_id({"metadata": {"program_id": ""}}, "program_id") is None
def test_first_user_text_reads_the_initial_request_including_text_parts():
assert first_user_text(_messages("hello")) == "hello"
parts = [
{
"role": "user",
"content": [{"type": "text", "text": "a"}, {"type": "image_url"}, {"type": "text", "text": "b"}],
}
]
assert first_user_text(parts) == "a\nb"
assert first_user_text([{"role": "assistant", "content": "x"}]) == ""
assert first_user_text(None) == ""
@pytest.mark.asyncio
async def test_first_request_binds_and_later_requests_reuse_the_binding():
router, maker = _router()
first = {"metadata": {"program_id": "t1", "user_api_key_hash": "key-1"}}
response = await router.async_pre_routing_hook(model="oracle", request_kwargs=first, messages=_messages())
assert response is not None and response.model == "smart"
assert (
first["metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart" and first["metadata"][PROGRAM_ID_METADATA_KEY] == "t1"
)
assert response.routing_decision["router_type"] == "oracle" and response.routing_decision["routed_model"] == "smart"
later = {"metadata": {"program_id": "t1", "user_api_key_hash": "key-1"}}
again = await router.async_pre_routing_hook(
model="oracle", request_kwargs=later, messages=_messages("now run the tests")
)
assert again is not None and again.model == "smart"
assert maker.selections == 1
assert router.binding("t1", "key-1").api_key_hash == "key-1"
other = await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "t2"}}, messages=_messages()
)
assert other.model == "fast" and router.programs_bound == 2
@pytest.mark.asyncio
async def test_concurrent_first_requests_of_one_program_share_a_single_binding():
class _SlowMaker(_RecordingMaker):
async def select(self, context: ProgramContext) -> str:
await asyncio.sleep(0.01) # a decision maker that does I/O (pre_routing, custom)
return await super().select(context)
router, maker = _router(maker=_SlowMaker())
responses = await asyncio.gather(
*(
router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "t1"}}, messages=_messages()
)
for _ in range(3)
)
)
assert {response.model for response in responses} == {"smart"}
assert maker.selections == 1 and router.programs_bound == 1 and router.binding("t1").model == "smart"
@pytest.mark.asyncio
async def test_a_non_finite_verifier_score_is_a_failed_verification():
class _NaN:
async def verify(self, outcome: ProgramOutcome) -> float:
return float("nan")
router, maker = _router(verifier=_NaN())
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
)
router.observe_request("t", cost=0.1, response_text="answer")
result = await router.complete("t")
assert result != result and router.verifications_failed == 1 and maker.updates == []
assert router.binding("t") is None # the slot was released all the same
@pytest.mark.asyncio
async def test_requests_without_a_program_id_route_statelessly_and_never_learn():
router, maker = _router()
kwargs = {"metadata": {}}
response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages())
assert response.model == "smart" and PROGRAM_ID_METADATA_KEY not in kwargs["metadata"]
assert router.stateless_requests == 1 and router.binding("") is None
assert router.complete("", score=1.0) is None
@pytest.mark.asyncio
async def test_complete_releases_the_slot_now_and_learns_in_verification_order():
router, maker = _router(verifier=_SlowVerifier({"t1": 0.05, "t2": 0.0}))
for program_id in ("t1", "t2"):
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages()
)
router.observe_request("t1", cost=0.4, response_text="patch")
router.observe_request("t1", cost=0.2, response_text="tests pass")
router.observe_request("t2", cost=0.1, response_text="wrong")
t1 = router.complete("t1", score=1.0)
t2 = router.complete("t2", score=0.0)
assert router.binding("t1") is None and router.binding("t2") is None
assert router.pending_verifications == 2
assert await t2 == 0.0
assert await t1 == 1.0
assert [u[0] for u in maker.updates] == ["t2", "t1"]
assert maker.updates[1] == ("t1", "smart", 1.0)
snapshot = await router.get_state_snapshot()
assert snapshot["programs_completed"] == 2 and snapshot["recent_accuracy"] == {"smart": 1.0, "fast": 0.0}
assert snapshot["recent_feedback"][-1]["cost"] == pytest.approx(0.6)
@pytest.mark.asyncio
async def test_a_full_verification_backlog_rejects_completion_and_keeps_the_program_bound():
router, maker = _router(verifier=_SlowVerifier({"t1": 0.05}), max_pending_verifications=1)
for program_id in ("t1", "t2"):
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages()
)
router.observe_request(program_id, cost=0.1, response_text="answer")
first = router.complete("t1", score=1.0)
with pytest.raises(VerificationBacklogFull, match="max_pending_verifications=1"):
router.complete("t2", score=0.0)
assert router.binding("t2") is not None and router.pending_verifications == 1
assert router.verifications_rejected == 1 and router.programs_completed == 1
assert await first == 1.0
assert await router.complete("t2", score=0.0) == 0.0
assert router.binding("t2") is None and [u[0] for u in maker.updates] == ["t1", "t2"]
assert (await router.get_state_snapshot())["verifications_rejected"] == 1
@pytest.mark.asyncio
async def test_a_verification_that_outlives_its_timeout_fails_and_teaches_nothing():
router, maker = _router(verifier=_SlowVerifier({"t": 5.0}), verification_timeout_seconds=0.01)
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
)
router.observe_request("t", cost=0.1, response_text="answer")
task = router.complete("t", score=1.0)
assert task is not None and math.isnan(await task)
assert router.verifications_failed == 1 and maker.updates == []
assert router.last_failure is not None and "verification: TimeoutError" in router.last_failure
assert router.pending_verifications == 0
@pytest.mark.asyncio
async def test_a_program_s_message_logging_opt_out_is_kept_on_its_context():
router, _ = _router()
await router.async_pre_routing_hook(
model="oracle",
request_kwargs={"metadata": {"program_id": "private", "turn_off_message_logging": True}},
messages=_messages(),
)
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "public"}}, messages=_messages()
)
await router.async_pre_routing_hook(
model="oracle",
request_kwargs={
"metadata": {"program_id": "by-header", "headers": {"litellm-enable-message-redaction": "true"}}
},
messages=_messages(),
)
assert router.binding("private").context.message_logging_off is True
assert router.binding("by-header").context.message_logging_off is True
assert router.binding("public").context.message_logging_off is False
@pytest.mark.asyncio
async def test_the_decision_maker_sees_the_caller_s_identity_and_none_of_the_request_s_other_state():
router, maker = _router()
kwargs = {
"metadata": {
"program_id": "t",
"user_api_key_hash": "hash-1",
"user_api_key_team_id": "team-1",
"user_api_key_budget_reservation": {"amount": 1.0},
"user_api_key_auth": {"api_key": "hash-1", "budget_reservation": {"amount": 1.0}},
"routing_decision": {"router_type": "oracle"},
"custom": "value",
},
"litellm_metadata": {"user_api_key_user_id": "user-1", "_autorouter_baseline_route": "stamp"},
}
await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages())
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"litellm_metadata": {"user_api_key_hash": "hash-2"}}, messages=_messages()
)
bound, stateless = maker.contexts
assert bound.caller == {
"user_api_key_hash": "hash-1",
"user_api_key_team_id": "team-1",
"user_api_key_user_id": "user-1",
"user_api_key_auth": {"api_key": "hash-1"},
"internal_call_origin": "autorouter_classifier",
}
assert budget_reservation_from_metadata(dict(bound.caller)) is None
assert stateless.caller == {"user_api_key_hash": "hash-2", "internal_call_origin": "autorouter_classifier"}
assert router.context_for("", "task", {"metadata": {"routing_decision": {}}}).caller == {
"internal_call_origin": "autorouter_classifier"
}
assert router.binding("t", "hash-1").context.caller == {}
@pytest.mark.asyncio
async def test_an_opt_out_on_a_later_turn_makes_the_whole_program_s_verification_private():
seen: list[tuple[bool, bool | None]] = []
class _Peek:
async def verify(self, outcome: ProgramOutcome) -> float:
inherited = initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging")
seen.append((outcome.message_logging_off, inherited))
return 1.0
router, _ = _router(verifier=_Peek())
for program_id in ("public", "late-private"):
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages()
)
router.observe_request("public", cost=0.1, response_text="a")
router.observe_request("late-private", cost=0.1, response_text="a")
router.mark_message_logging_off("late-private")
router.observe_request("late-private", cost=0.1, response_text="c")
assert router.binding("public").message_logging_off is False
assert router.binding("late-private").message_logging_off is True
assert await router.complete("public", score=1.0) == 1.0
assert await router.complete("late-private", score=1.0) == 1.0
assert seen == [(False, None), (True, True)]
assert initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging") is None
@pytest.mark.asyncio
async def test_a_responses_api_request_binds_its_program_on_the_input_text():
router, _ = _router()
response = await router.async_pre_routing_hook(
model="oracle",
request_kwargs={"metadata": {"program_id": "r"}},
messages=None,
input=[{"role": "user", "content": [{"type": "input_text", "text": "write a limerick"}]}],
)
assert response.messages is None and router.binding("r").context.prompt == "write a limerick"
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "s"}}, messages=None, input="plain text"
)
assert router.binding("s").context.prompt == "plain text"
@pytest.mark.asyncio
async def test_complete_cost_override_and_payload_reach_the_verifier():
seen: list[ProgramOutcome] = []
class _Capture:
async def verify(self, outcome: ProgramOutcome) -> float:
seen.append(outcome)
return 1.0
router, maker = _router(verifier=_Capture())
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages("do x")
)
transcript = [{"role": "user", "content": "do x"}, {"role": "assistant", "content": "answer"}]
router.observe_request("t", cost=0.3, response_text="answer", messages=transcript)
await router.complete("t", cost=2.5, payload={"container": "c1"})
assert seen[0].cost == 2.5 and seen[0].payload == {"container": "c1"} and seen[0].response_text == "answer"
assert list(seen[0].messages) == transcript
assert seen[0].prompt == "do x" and maker.updates == [("t", "smart", 1.0)]
@pytest.mark.asyncio
async def test_failed_verification_is_counted_and_does_not_update():
class _Broken:
async def verify(self, outcome):
raise RuntimeError("sandbox down")
router, maker = _router(verifier=_Broken())
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
)
router.observe_request("t", cost=0.1, response_text="answer")
result = await router.complete("t")
assert result != result and router.verifications_failed == 1 and maker.updates == []
snapshot = await router.get_state_snapshot()
assert (
snapshot["last_failure"] == "verification: RuntimeError: sandbox down"
) # kept for the state endpoint, not logged
@pytest.mark.asyncio
async def test_expired_programs_are_evicted_when_the_table_is_full():
now = [1000.0]
router, maker = _router(clock=lambda: now[0], program_ttl_seconds=10, max_programs=3)
for program_id in ("a", "b", "c"):
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages()
)
now[0] += 4.0
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "d"}}, messages=_messages()
)
assert router.binding("a") is None and router.binding("b") is not None and router.binding("d") is not None
assert router.programs_evicted == 1
@pytest.mark.asyncio
async def test_decision_is_stamped_in_the_proxy_internal_bucket_when_one_exists():
router, _ = _router()
kwargs = {"metadata": {"program_id": "t1"}, "litellm_metadata": {"user_api_key_hash": "k"}}
response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages())
assert response.model == "smart"
assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart"
assert CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"]
assert router.binding("t1", "k").api_key_hash == "k"
@pytest.mark.asyncio
async def test_the_same_program_id_under_another_key_is_another_program():
router, maker = _router()
for key in ("key-a", "key-b"):
await router.async_pre_routing_hook(
model="oracle",
request_kwargs={"metadata": {"program_id": "t1", "user_api_key_hash": key}},
messages=_messages(),
)
assert router.programs_bound == 2 and maker.selections == 2
assert router.binding("t1") is None # no key: not the same program either
assert [binding.api_key_hash for binding in router.bindings_named("t1")] == ["key-a", "key-b"]
router.observe_request("t1", cost=0.1, response_text="answer", owner="key-b")
router.complete("t1", score=1.0, owner="key-b")
assert router.binding("t1", "key-a") is not None and router.binding("t1", "key-b") is None
await router.drain()
@pytest.mark.asyncio
async def test_completion_before_any_observed_response_is_deferred_to_the_next_one():
router, maker = _router()
await router.async_pre_routing_hook(
model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
)
assert router.complete("t", score=1.0, cost=2.0, payload={"run": "r1"}) is None # nothing answered yet
binding = router.binding("t")
assert binding is not None and binding.deferred is not None and router.programs_completed == 0
assert maker.updates == []
router.observe_request("t", cost=0.3, response_text="answer") # the response the feedback raced
assert router.binding("t") is None and router.programs_completed == 1
await router.drain()
assert maker.updates == [("t", "smart", 1.0)]
@pytest.mark.asyncio
async def test_pre_routing_hook_replaces_caller_supplied_internal_keys():
router, _ = _router()
kwargs = {
"metadata": {"program_id": "t1", PROGRAM_ID_METADATA_KEY: "victim", CHOSEN_MODEL_METADATA_KEY: "fast"},
"litellm_metadata": {PROGRAM_ID_METADATA_KEY: "victim"},
}
response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages())
assert PROGRAM_ID_METADATA_KEY not in kwargs["metadata"] and CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"]
assert kwargs["litellm_metadata"][PROGRAM_ID_METADATA_KEY] == "t1"
assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == response.model
stateless = {"metadata": {PROGRAM_ID_METADATA_KEY: "victim", CHOSEN_MODEL_METADATA_KEY: "fast"}}
await router.async_pre_routing_hook(model="oracle", request_kwargs=stateless, messages=_messages())
assert PROGRAM_ID_METADATA_KEY not in stateless["metadata"]
assert stateless["metadata"][CHOSEN_MODEL_METADATA_KEY] in ("smart", "fast")

View file

@ -0,0 +1,256 @@
"""Router-level wiring of `auto_router/oracle_router`: registration, deferred init, dispatch, hooks, removal."""
import pytest
import litellm
from litellm import Router
from litellm.router_strategy.oracle_router.config import PROGRAM_ID_METADATA_KEY
from litellm.router_strategy.oracle_router.decision import PreRoutingDecisionMaker, ProgramContext
from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook
from litellm.types.router import Deployment, LiteLLM_Params, RequestType
def _oracle(router: Router, name: str):
return router.registry().plugins("oracle")[name][0].strategy
def _model_list(decision_maker=None, verifier=None, extra=()):
config = {"available_models": ["smart", "fast"]}
if decision_maker:
config["decision_maker"] = decision_maker
if verifier:
config["verifier"] = verifier
return [
{
"model_name": "oracle",
"litellm_params": {"model": "auto_router/oracle_router", "oracle_router_config": config},
},
{"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}},
{"model_name": "fast", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}},
*extra,
]
def _oracle_hooks():
return litellm.logging_callback_manager.get_custom_loggers_for_type(OracleRouterPostCallHook)
class _CustomMaker:
def __init__(self, models):
self.models = tuple(models)
async def select(self, context):
return self.models[-1]
def update(self, context, model, score):
pass
def snapshot(self):
return {"type": "custom"}
def test_prefix_is_classified_as_oracle_and_not_as_a_semantic_router():
router = Router(model_list=[])
params = LiteLLM_Params(model="auto_router/oracle_router")
assert router._is_oracle_router_deployment(litellm_params=params) is True
assert router._is_auto_router_deployment(litellm_params=params) is False
assert router._is_adaptive_router_deployment(litellm_params=params) is False
assert router._is_oracle_router_deployment(litellm_params=LiteLLM_Params(model="openai/gpt-4o")) is False
def test_init_registers_the_router_and_exactly_one_hook():
router = Router(model_list=_model_list())
assert "oracle" in router.registry().plugins("oracle")
assert _oracle(router, "oracle").models == ("smart", "fast")
hooks = [hook for hook in _oracle_hooks() if hook.oracle_router is _oracle(router, "oracle")]
assert len(hooks) == 1
def test_init_without_config_raises():
with pytest.raises(ValueError, match="oracle_router_config is required"):
Router(model_list=[{"model_name": "oracle", "litellm_params": {"model": "auto_router/oracle_router"}}])
@pytest.mark.asyncio
async def test_pre_routing_hook_dispatches_to_the_oracle_router_and_binds_the_program():
router = Router(model_list=_model_list(decision_maker={"type": "fixed", "model": "fast"}))
kwargs = {"metadata": {"program_id": "task-1"}}
response = await router.async_pre_routing_hook(
model="oracle", request_kwargs=kwargs, messages=[{"role": "user", "content": "fix the bug"}]
)
assert response is not None and response.model == "fast"
assert response.routing_decision["router_type"] == "oracle"
assert kwargs["metadata"][PROGRAM_ID_METADATA_KEY] == "task-1"
assert _oracle(router, "oracle").binding("task-1").model == "fast"
def test_pre_routing_decision_maker_resolves_a_strategy_router_listed_after_it():
quality = {
"model_name": "tiered",
"litellm_params": {
"model": "auto_router/quality_router",
"quality_router_default_model": "fast",
"quality_router_config": {"available_models": ["smart", "fast"]},
},
}
model_list = _model_list(decision_maker={"type": "pre_routing", "router": "tiered"}, extra=(quality,))
model_list[1]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 3}}
model_list[2]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 1}}
router = Router(model_list=model_list)
assert isinstance(_oracle(router, "oracle").decision_maker, PreRoutingDecisionMaker)
def test_init_reads_adaptive_router_preferences_as_bandit_priors():
model_list = _model_list()
model_list[1]["model_info"] = {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}}
router = Router(model_list=model_list)
maker = _oracle(router, "oracle").decision_maker
ctx = ProgramContext(program_id="p", prompt="x", request_type=RequestType.GENERAL)
assert maker.estimate(ctx, "smart") > maker.estimate(ctx, "fast")
def test_pre_routing_decision_maker_with_unknown_router_fails_init():
with pytest.raises(ValueError, match="not a configured strategy router"):
Router(model_list=_model_list(decision_maker={"type": "pre_routing", "router": "ghost"}))
def test_hot_reload_keeps_the_existing_router_and_its_learned_state():
router = Router(model_list=_model_list())
before = _oracle(router, "oracle")
router.set_model_list(_model_list())
assert _oracle(router, "oracle") is before
assert len([hook for hook in _oracle_hooks() if hook.oracle_router is before]) == 1
def test_deleting_the_deployment_unregisters_the_router_and_its_hook():
router = Router(model_list=_model_list())
before = _oracle(router, "oracle")
router.delete_deployment(id=router.get_model_ids(model_name="oracle")[0])
assert "oracle" not in router.registry().plugins("oracle")
assert all(hook.oracle_router is not before for hook in _oracle_hooks())
def test_deployment_input_costs_prefer_litellm_params_over_model_info_and_skip_unknown_models():
model_list = _model_list()
model_list[1]["litellm_params"]["input_cost_per_token"] = 2e-6
model_list[1]["model_info"] = {"input_cost_per_token": 9e-6}
model_list[2]["model_info"] = {"input_cost_per_token": 1e-7}
router = Router(model_list=model_list)
assert router._deployment_input_costs(["smart", "fast", "ghost"]) == {"smart": 2e-6, "fast": 1e-7}
def test_deployment_adaptive_prefs_read_model_info_where_declared():
model_list = _model_list()
model_list[1]["model_info"] = {"adaptive_router_preferences": {"quality_tier": 3, "strengths": ["code_generation"]}}
router = Router(model_list=model_list)
prefs = router._deployment_adaptive_prefs(["smart", "fast", "ghost"])
assert set(prefs) == {"smart"} and prefs["smart"].quality_tier == 3
assert prefs["smart"].strengths == [RequestType.CODE_GENERATION]
def test_resolve_strategy_router_finds_registered_strategy_routers_only():
quality = {
"model_name": "tiered",
"litellm_params": {
"model": "auto_router/quality_router",
"quality_router_default_model": "fast",
"quality_router_config": {"available_models": ["smart", "fast"]},
},
}
model_list = _model_list(extra=(quality,))
model_list[1]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 3}}
model_list[2]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 1}}
router = Router(model_list=model_list)
assert router._resolve_strategy_router("tiered") is router.registry().plugins("quality")["tiered"][0].strategy
assert router._resolve_strategy_router("smart") is None and router._resolve_strategy_router("ghost") is None
def test_init_oracle_router_deployment_registers_the_router_and_finalize_is_idempotent():
router = Router(model_list=_model_list()[1:]) # the two plain deployments, no oracle router yet
deployment = Deployment(
model_name="late-oracle",
litellm_params=LiteLLM_Params(
model="auto_router/oracle_router", oracle_router_config={"available_models": ["smart", "fast"]}
),
)
router.init_oracle_router_deployment(deployment=deployment)
oracle = _oracle(router, "late-oracle")
assert oracle.models == ("smart", "fast") and oracle.router_name == "late-oracle"
router._sync_oracle_router_hooks()
router._sync_oracle_router_hooks()
assert len([hook for hook in _oracle_hooks() if hook.oracle_router is oracle]) == 1
router._finalize_oracle_router_if_configured() # nothing new in model_list: the registry is untouched
assert _oracle(router, "late-oracle") is oracle
with pytest.raises(ValueError, match="oracle_router_config is required"):
router.init_oracle_router_deployment(
deployment=Deployment(model_name="bare", litellm_params=LiteLLM_Params(model="auto_router/oracle_router"))
)
def test_upserting_an_oracle_deployment_builds_its_router_once_and_rebuilds_it_when_its_config_changes():
router = Router(model_list=_model_list()[1:]) # the two plain deployments, no oracle router yet
deployment = Deployment(
model_name="late-oracle",
litellm_params=LiteLLM_Params(
model="auto_router/oracle_router", oracle_router_config={"available_models": ["smart", "fast"]}
),
model_info={"id": "oracle-1"},
)
router.upsert_deployment(deployment=deployment) # a new id: nothing was on the router before
oracle = _oracle(router, "late-oracle")
assert oracle.models == ("smart", "fast")
router.upsert_deployment(deployment=deployment) # unchanged: the router and its learned state survive
assert _oracle(router, "late-oracle") is oracle
assert len([hook for hook in _oracle_hooks() if hook.oracle_router is oracle]) == 1
router.upsert_deployment(
deployment=Deployment(
model_name="late-oracle",
litellm_params=LiteLLM_Params(
model="auto_router/oracle_router", oracle_router_config={"available_models": ["fast"]}
),
model_info={"id": "oracle-1"},
)
)
rebuilt = _oracle(router, "late-oracle")
assert rebuilt is not oracle and rebuilt.models == ("fast",)
assert [hook.oracle_router for hook in _oracle_hooks() if hook.oracle_router in (oracle, rebuilt)] == [rebuilt]
def test_custom_plugins_load_from_the_config_file_but_never_for_models_written_through_the_api(monkeypatch):
import subprocess
popen_calls = []
monkeypatch.setattr(subprocess, "Popen", lambda *args, **kwargs: popen_calls.append(args))
from_config = Router(model_list=_model_list(decision_maker={"type": "custom", "path": f"{__name__}:_CustomMaker"}))
assert _oracle(from_config, "oracle").decision_maker.models == ("smart", "fast")
for config_key in ("decision_maker", "verifier"):
model_list = _model_list(**{config_key: {"type": "custom", "path": "subprocess:Popen"}})
model_list[0]["model_info"] = {"db_model": True} # what the proxy sets for /model/new and /model/update
model_list[0]["litellm_params"]["oracle_router_config"]["available_models"] = ["/bin/sh", "-c", "id"]
with pytest.raises(ValueError, match="config file"):
Router(model_list=model_list)
router = Router(model_list=_model_list()[1:])
with pytest.raises(ValueError, match="config file"):
router.upsert_deployment(
deployment=Deployment(
model_name="api-oracle",
litellm_params=LiteLLM_Params(
model="auto_router/oracle_router",
oracle_router_config={
"available_models": ["smart", "fast"],
"verifier": {"type": "custom", "path": "subprocess:Popen"},
},
),
model_info={"id": "api-oracle-1", "db_model": True},
)
)
assert popen_calls == [] and "api-oracle" not in router.registry().plugins("oracle")
def test_strategy_router_dependencies_cover_available_models():
from litellm.router_strategy.auto_router.auto_router_model_naming import strategy_router_dependencies
deps = strategy_router_dependencies(
{"model": "auto_router/oracle_router", "oracle_router_config": {"available_models": ["smart", "fast"]}}
)
assert {(dep.model_name, dep.role) for dep in deps} == {("smart", "tier"), ("fast", "tier")}

View file

@ -0,0 +1,311 @@
"""Verifiers: LiteLLM guardrails and reported scores become the number the decision maker learns from."""
import pytest
from fastapi import HTTPException
from litellm.constants import REDACTED_BY_LITELLM
from litellm.exceptions import GuardrailRaisedException
from litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge import LLMAsAJudgeGuardrail
from litellm.router_strategy.oracle_router.verifier import (
GuardrailVerifier,
ProgramOutcome,
ReportedVerifier,
Verifier,
build_verifier,
clamp_score,
recorded_status,
response_text,
)
from litellm.types.router import OracleVerifierConfig
from litellm.types.utils import Choices, Message, ModelResponse
def _outcome(score=None, cost=0.0, answer="done", messages=()) -> ProgramOutcome:
return ProgramOutcome(
program_id="p", model="smart", prompt="task", response_text=answer, cost=cost, messages=messages, score=score
)
@pytest.mark.asyncio
async def test_reported_verifier_uses_the_reported_score_or_its_default():
assert await ReportedVerifier().verify(_outcome(score=0.8)) == 0.8
assert await ReportedVerifier().verify(_outcome(score=4.0)) == 1.0
assert await ReportedVerifier(default=0.25).verify(_outcome()) == 0.25
class _Guardrail:
"""A guardrail that blocks the way LiteLLM's do: by raising, with no status record."""
def __init__(self, blocks: bool) -> None:
self.blocks = blocks
self.seen: list[dict] = []
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
self.seen.append({"inputs": inputs, "input_type": input_type, "request_data": request_data})
if self.blocks:
raise GuardrailRaisedException(guardrail_name="strict", message="blocked", blocked_content=True)
return inputs
class _RecordingGuardrail:
"""A guardrail that reports through LiteLLM's status record, like llm_as_a_judge, and may also raise."""
def __init__(self, status: str, error: Exception | None = None, name: str = "judge") -> None:
self.status, self.error, self.name = status, error, name
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
self.seen = dict(request_data["metadata"]) # what the guardrail was handed, before it records anything
records = request_data["metadata"].setdefault("standard_logging_guardrail_information", [])
records.append({"guardrail_name": self.name, "guardrail_status": self.status}) # appended, like the real ones
if self.error is not None:
raise self.error
return inputs
@pytest.mark.asyncio
async def test_feedback_payload_cannot_pose_as_the_proxy_to_the_guardrail():
judge = _RecordingGuardrail("success")
outcome = ProgramOutcome(
program_id="p",
model="smart",
prompt="task",
response_text="answer",
cost=0.1,
payload={
"container": "c1",
"user_api_key_team_id": "someone-else",
"user_api_key": "sk-forged",
"internal_call_origin": "shadow_eval_judge",
"headers": {"litellm-disable-message-redaction": "true"},
"routing_decision": {"router_type": "oracle"},
},
)
assert await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(outcome) == 1.0
assert judge.seen == {"container": "c1", "program_id": "p"}
@pytest.mark.asyncio
async def test_guardrail_verifier_refuses_a_final_answer_litellm_redacted_instead_of_judging_the_placeholder():
judge = _Guardrail(blocks=False)
outcome = ProgramOutcome(program_id="p", model="smart", prompt="task", response_text=REDACTED_BY_LITELLM, cost=0.1)
with pytest.raises(RuntimeError, match="redacted before verification"):
await GuardrailVerifier("ok", lookup={"ok": judge}.get).verify(outcome)
stale_answer_behind_a_redacted_turn = ProgramOutcome(
program_id="p",
model="smart",
prompt="task",
response_text="an earlier public answer",
cost=0.1,
messages=[{"role": "user", "content": REDACTED_BY_LITELLM}],
)
with pytest.raises(RuntimeError, match="redacted before verification"):
await GuardrailVerifier("ok", lookup={"ok": judge}.get).verify(stale_answer_behind_a_redacted_turn)
assert judge.seen == []
@pytest.mark.asyncio
async def test_guardrail_verifier_hands_the_judge_the_transcript_and_the_feedback_payload():
passing, blocking = _Guardrail(blocks=False), _Guardrail(blocks=True)
lookup = {"ok": passing, "strict": blocking}.get
transcript = ({"role": "user", "content": "task"}, {"role": "assistant", "content": "final answer"})
outcome = ProgramOutcome(
program_id="p",
model="smart",
prompt="task",
response_text="final answer",
cost=0.0,
messages=transcript,
payload={"reference": "42"},
)
assert await GuardrailVerifier("ok", lookup=lookup).verify(outcome) == 1.0
assert passing.seen == [
{
"inputs": {"texts": ["final answer"], "model": "smart", "structured_messages": list(transcript)},
"input_type": "response",
"request_data": {"model": "smart", "metadata": {"program_id": "p", "reference": "42"}},
}
]
assert await GuardrailVerifier("strict", lookup=lookup).verify(outcome) == 0.0
with pytest.raises(LookupError):
await GuardrailVerifier("missing", lookup=lookup).verify(outcome)
@pytest.mark.asyncio
async def test_guardrail_verifier_reads_the_recorded_verdict_before_the_exception():
judge = {
"passed": _RecordingGuardrail("success", name="passed"),
"logged": _RecordingGuardrail("guardrail_intervened", name="logged"), # on_failure: log -> no exception
"blocked": _RecordingGuardrail(
"guardrail_intervened", HTTPException(status_code=422, detail="judge"), "blocked"
),
"flagged": _RecordingGuardrail("guardrail_flagged", name="flagged"),
}
assert await GuardrailVerifier("passed", lookup=judge.get).verify(_outcome()) == 1.0
assert await GuardrailVerifier("logged", lookup=judge.get).verify(_outcome()) == 0.0
assert await GuardrailVerifier("blocked", lookup=judge.get).verify(_outcome()) == 0.0
assert await GuardrailVerifier("flagged", lookup=judge.get).verify(_outcome()) == 0.0
@pytest.mark.asyncio
async def test_a_program_without_response_text_is_a_failed_verification_not_a_pass():
judge = _RecordingGuardrail("success", name="judge") # would skip an empty answer without a record
with pytest.raises(RuntimeError, match="no response text"):
await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(_outcome(answer=" "))
assert not hasattr(judge, "seen") # the guardrail was never invoked
@pytest.mark.asyncio
async def test_feedback_payload_cannot_pre_seed_the_status_record_or_rename_the_program():
judge = _RecordingGuardrail("guardrail_intervened", name="judge") # on_failure: log -> no exception
forged = {"standard_logging_guardrail_information": [0], "program_id": "someone-else", "container": "c1"}
outcome = ProgramOutcome(
program_id="p", model="smart", prompt="task", response_text="wrong", cost=0.0, messages=(), payload=forged
)
assert await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(outcome) == 0.0
assert judge.seen == {"program_id": "p", "container": "c1"}
@pytest.mark.asyncio
async def test_guardrail_that_could_not_evaluate_is_a_failed_verification_not_a_pass():
judge = {
"down": _RecordingGuardrail("guardrail_failed_to_respond", name="down"), # llm_as_a_judge fails open
"skipped": _RecordingGuardrail("not_run", name="skipped"),
"crashed": _RecordingGuardrail("success", RuntimeError("boom"), "crashed"),
}
with pytest.raises(RuntimeError, match="did not evaluate"):
await GuardrailVerifier("down", lookup=judge.get).verify(_outcome())
with pytest.raises(RuntimeError, match="not_run"):
await GuardrailVerifier("skipped", lookup=judge.get).verify(_outcome())
class _NoVerdict(_Guardrail):
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
raise GuardrailRaisedException(guardrail_name="g", message="timeout", blocked_content=False)
with pytest.raises(GuardrailRaisedException):
await GuardrailVerifier("g", lookup={"g": _NoVerdict(blocks=True)}.get).verify(_outcome())
assert await GuardrailVerifier("crashed", lookup=judge.get).verify(_outcome()) == 1.0 # the record wins
@pytest.mark.asyncio
async def test_guardrail_exceptions_without_a_record_are_classified():
class _Raises:
def __init__(self, error: Exception) -> None:
self.error = error
async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None):
raise self.error
lookup = {
"http": _Raises(HTTPException(status_code=400, detail="Violated guardrail policy")),
"infra": _Raises(ConnectionError("provider down")),
}.get
assert await GuardrailVerifier("http", lookup=lookup).verify(_outcome()) == 0.0
with pytest.raises(ConnectionError):
await GuardrailVerifier("infra", lookup=lookup).verify(_outcome())
def _judge(monkeypatch, on_failure: str, verdict) -> LLMAsAJudgeGuardrail:
"""The real llm_as_a_judge guardrail with only its judge-model call stubbed."""
guardrail = LLMAsAJudgeGuardrail(
guardrail_name="answer-judge",
judge_model="judge",
criteria=[{"name": "task_completed", "weight": 100, "description": "the task was completed"}],
overall_threshold=50,
on_failure=on_failure,
)
async def run_judge(messages, text_under_review, input_type="response"):
if isinstance(verdict, Exception):
raise verdict
return verdict
monkeypatch.setattr(guardrail, "_run_judge", run_judge)
return guardrail
@pytest.mark.asyncio
@pytest.mark.parametrize(
("on_failure", "overall_score", "expected"),
[
("block", 90, 1.0), # passes the threshold: recorded as success
("block", 10, 0.0), # fails: the judge raises 422 and records guardrail_intervened
("log", 10, 0.0), # fails but only logs: no exception, still recorded as guardrail_intervened
],
)
async def test_real_llm_as_a_judge_verdicts_become_scores(monkeypatch, on_failure, overall_score, expected):
guardrail = _judge(monkeypatch, on_failure, {"overall_score": overall_score, "verdicts": []})
verifier = GuardrailVerifier("answer-judge", lookup={"answer-judge": guardrail}.get)
assert await verifier.verify(_outcome(answer="final answer")) == expected
@pytest.mark.asyncio
async def test_real_llm_as_a_judge_that_fails_open_is_not_a_pass(monkeypatch):
guardrail = _judge(monkeypatch, "block", RuntimeError("judge model unreachable")) # the guardrail returns normally
verifier = GuardrailVerifier("answer-judge", lookup={"answer-judge": guardrail}.get)
with pytest.raises(RuntimeError, match="did not evaluate"):
await verifier.verify(_outcome(answer="final answer"))
def test_recorded_status_reads_both_buckets_and_takes_the_latest():
data = {
"metadata": {
"standard_logging_guardrail_information": [{"guardrail_name": "a", "guardrail_status": "success"}]
},
"litellm_metadata": {
"standard_logging_guardrail_information": {"guardrail_name": "a", "guardrail_status": "guardrail_flagged"}
},
}
assert recorded_status(data, "a") == "success" # a single dict, not a list, is not a valid record set
data["litellm_metadata"]["standard_logging_guardrail_information"] = [
{"guardrail_name": "a", "guardrail_status": "guardrail_intervened"}
]
assert recorded_status(data, "a") == "guardrail_intervened"
assert recorded_status(data, "other") is None
assert recorded_status({"metadata": "junk"}, "a") is None
def test_clamp_score_rejects_non_finite_scores():
assert clamp_score(1.7) == 1.0 and clamp_score(-2) == 0.0
for bad in (float("nan"), float("inf")):
with pytest.raises(ValueError, match="finite"):
clamp_score(bad)
@pytest.mark.asyncio
async def test_reported_verifier_rejects_a_nan_score():
with pytest.raises(ValueError, match="finite"):
await ReportedVerifier().verify(_outcome(score=float("nan")))
@pytest.mark.asyncio
async def test_guardrail_verifier_without_a_transcript_sends_the_initial_request():
passing = _Guardrail(blocks=False)
await GuardrailVerifier("ok", lookup={"ok": passing}.get).verify(_outcome())
assert passing.seen[0]["inputs"]["structured_messages"] == [{"role": "user", "content": "task"}]
def test_response_text_reads_chat_completions_only():
assert response_text(ModelResponse(choices=[Choices(message=Message(content="hi", role="assistant"))])) == "hi"
assert response_text({"choices": []}) == ""
assert response_text(None) == ""
class CustomVerifier:
async def verify(self, outcome: ProgramOutcome) -> float:
return 0.5
class NotAVerifier:
pass
@pytest.mark.asyncio
async def test_build_verifier_by_type():
assert isinstance(build_verifier(OracleVerifierConfig(type="reported")), ReportedVerifier)
assert isinstance(build_verifier(OracleVerifierConfig(type="guardrail", guardrail="g")), GuardrailVerifier)
custom = build_verifier(OracleVerifierConfig(type="custom", path=f"{__name__}:CustomVerifier"))
assert isinstance(custom, Verifier) and await custom.verify(_outcome()) == 0.5
with pytest.raises(TypeError):
build_verifier(OracleVerifierConfig(type="custom", path=f"{__name__}:NotAVerifier"))
with pytest.raises(ValueError, match=r"verifier\.guardrail"):
OracleVerifierConfig(type="guardrail")

View file

@ -11651,6 +11651,51 @@ export interface paths {
patch: operations["openrouter_proxy_route_openrouter__endpoint__patch"];
trace?: never;
};
"/oracle_router/feedback": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
get?: never;
put?: never;
/**
* Submit Oracle Router Feedback
* @description Complete a program: its model slot is released now, its verifier runs in the background.
*
* Programs are private to the key that started them; an admin may complete any key's program. 404 when
* the caller holds no program with that id, which also covers programs completed earlier. A program none
* of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed
* response completes it with this feedback.
*/
post: operations["submit_oracle_router_feedback_oracle_router_feedback_post"];
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/oracle_router/state": {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
/**
* Get Oracle Router State
* @description Decision-maker state, active bindings and recent verified feedback for every ORACLE router. Admin only.
*/
get: operations["get_oracle_router_state_oracle_router_state_get"];
put?: never;
post?: never;
delete?: never;
options?: never;
head?: never;
patch?: never;
trace?: never;
};
"/organization/daily/activity": {
parameters: {
query?: never;
@ -36100,6 +36145,10 @@ export interface components {
openai_identity_token_file?: string | null;
/** Openai Service Account Id */
openai_service_account_id?: string | null;
/** Oracle Router Config */
oracle_router_config?: {
[key: string]: unknown;
} | null;
/** Organization */
organization?: string | null;
/** Otpm */
@ -40652,6 +40701,52 @@ export interface components {
} & {
[key: string]: unknown;
};
/**
* OracleRouterFeedbackRequest
* @description Report that a program finished. ``score`` is optional when the router's verifier grades it.
*/
OracleRouterFeedbackRequest: {
/**
* Cost
* @description Overrides the spend LiteLLM accumulated for the program
*/
cost?: number | null;
/**
* Payload
* @description Passed to the verifier unchanged
*/
payload?: {
[key: string]: unknown;
};
/** Program Id */
program_id: string;
/** Score */
score?: number | null;
};
/** OracleRouterFeedbackResponse */
OracleRouterFeedbackResponse: {
/**
* Deferred
* @description No response of the program has been observed yet; its next one completes it with this feedback
* @default false
*/
deferred: boolean;
/** Model */
model: string;
/** Pending Verifications */
pending_verifications: number;
/** Program Id */
program_id: string;
/** Router Name */
router_name: string;
};
/** OracleRouterStateResponse */
OracleRouterStateResponse: {
/** Routers */
routers: {
[key: string]: unknown;
}[];
};
/** OrgMember */
OrgMember: {
/**
@ -46637,7 +46732,7 @@ export interface components {
* Router Type
* @enum {string}
*/
router_type?: "complexity" | "adaptive" | "quality" | "semantic";
router_type?: "complexity" | "adaptive" | "quality" | "semantic" | "oracle";
/** Savings Baseline Deployment Id */
savings_baseline_deployment_id?: string;
/** Savings Baseline Model */
@ -51783,6 +51878,10 @@ export interface components {
openai_identity_token_file?: string | null;
/** Openai Service Account Id */
openai_service_account_id?: string | null;
/** Oracle Router Config */
oracle_router_config?: {
[key: string]: unknown;
} | null;
/** Organization */
organization?: string | null;
/** Otpm */
@ -68875,6 +68974,59 @@ export interface operations {
};
};
};
submit_oracle_router_feedback_oracle_router_feedback_post: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody: {
content: {
"application/json": components["schemas"]["OracleRouterFeedbackRequest"];
};
};
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["OracleRouterFeedbackResponse"];
};
};
/** @description Validation Error */
422: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["HTTPValidationError"];
};
};
};
};
get_oracle_router_state_oracle_router_state_get: {
parameters: {
query?: never;
header?: never;
path?: never;
cookie?: never;
};
requestBody?: never;
responses: {
/** @description Successful Response */
200: {
headers: {
[name: string]: unknown;
};
content: {
"application/json": components["schemas"]["OracleRouterStateResponse"];
};
};
};
};
get_organization_daily_activity_organization_daily_activity_get: {
parameters: {
query?: {