diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index c2740ac1d41..44b06a26154 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -123,6 +123,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( # Realtime / streaming "/v1/realtime", "/realtime", + # ORACLE router feedback and state: the program bindings live in the serving process + "/oracle_router/", "/v1/live/sessions", "/live/sessions", # Health & ops diff --git a/litellm/proxy/example_config_yaml/oracle_router_example.yaml b/litellm/proxy/example_config_yaml/oracle_router_example.yaml new file mode 100644 index 00000000000..f8d64bde8e1 --- /dev/null +++ b/litellm/proxy/example_config_yaml/oracle_router_example.yaml @@ -0,0 +1,51 @@ +# ORACLE router: one model per agentic program, learned from verified outcomes. +# +# Clients call `model: oracle-router` and tag every request of a task with the same program id: +# POST /v1/chat/completions { "model": "oracle-router", "metadata": { "program_id": "task-17" }, ... } +# The first request binds the program to a model; later requests reuse it. On the last request add +# "metadata": { "program_id": "task-17", "program_done": true, "program_score": 1.0 } +# or POST /oracle_router/feedback { "program_id": "task-17", "score": 1.0 } once the task is graded. +# +# Required env vars: OPENAI_API_KEY, LITELLM_MASTER_KEY. + +model_list: + - model_name: oracle-router + litellm_params: + model: auto_router/oracle_router + oracle_router_config: + available_models: ["smart", "fast"] # strongest first + decision_maker: + type: thompson # fixed | pre_routing | custom + weights: { quality: 0.7, cost: 0.3 } # the adaptive router's quality/cost trade-off + verifier: + type: guardrail # reported | custom + guardrail: answer-judge + + # Let LiteLLM's complexity router propose the model and have ORACLE bind, verify and report it: + # decision_maker: { type: pre_routing, router: tiered-router } + + - model_name: smart + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + + - model_name: fast + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: answer-judge + litellm_params: + guardrail: llm_as_a_judge + mode: post_call + default_on: false + judge_model: smart + overall_threshold: 50 + criteria: + - name: task_completed + weight: 100 + description: The assistant completed the user's task correctly, judged from the whole conversation. + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 2c7453ebb1c..faed652dde3 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -3970,7 +3970,7 @@ async def clear_cache() -> ReconcileOutcome: config_models: Final = [] db_model_ids: Final = [] - db_router_names: Final = set() + db_router_names: Final[set[str]] = set() # mutable-ok: filled in the classification pass below for model in current_models: model_info = model.get("model_info", {}) @@ -3979,7 +3979,7 @@ async def clear_cache() -> ReconcileOutcome: # Auto-router deployments (and only those) are wiped here, in the # same pass, so the reload rebuilds them -- see the comment below. model_name = model.get("model_name") - if model_name is not None and str(model.get("litellm_params", {}).get("model", "")).startswith( + if isinstance(model_name, str) and str(model.get("litellm_params", {}).get("model", "")).startswith( "auto_router/" ): db_router_names.add(model_name) @@ -4028,6 +4028,7 @@ async def clear_cache() -> ReconcileOutcome: llm_router.registry().plugins("complexity").pop(model_name, None) llm_router.registry().plugins("adaptive").pop(model_name, None) llm_router.registry().plugins("quality").pop(model_name, None) + llm_router.registry().plugins("oracle").pop(model_name, None) # Reload only DB models. _add_deployment_locked, not add_deployment: this # coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not diff --git a/litellm/proxy/management_endpoints/oracle_router_endpoints.py b/litellm/proxy/management_endpoints/oracle_router_endpoints.py new file mode 100644 index 00000000000..2361d06ea13 --- /dev/null +++ b/litellm/proxy/management_endpoints/oracle_router_endpoints.py @@ -0,0 +1,117 @@ +""" +ORACLE ROUTER MANAGEMENT ENDPOINTS + +POST /oracle_router/feedback - Report a finished program so its verifier runs and the router learns +GET /oracle_router/state - Live decision-maker state, bindings and recent feedback per router +""" + +from itertools import chain +from typing import TYPE_CHECKING, Annotated, Final + +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.router_strategy.oracle_router.oracle_router import VerificationBacklogFull +from litellm.types.management_endpoints.oracle_router_endpoints import ( + OracleRouterFeedbackRequest, + OracleRouterFeedbackResponse, + OracleRouterStateResponse, +) + +if TYPE_CHECKING: + from fastapi import APIRouter, Depends, HTTPException + + from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, ProgramBinding +else: + try: + from fastapi import APIRouter, Depends, HTTPException + except ImportError: + pass + +router: Final = APIRouter() + + +def _is_admin(user_api_key_dict: UserAPIKeyAuth) -> bool: + return user_api_key_dict.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + + +def _oracle_routers() -> tuple["OracleRouter", ...]: + from litellm.proxy.proxy_server import llm_router + + registry: Final = llm_router.registry().plugins("oracle") if llm_router is not None else {} + tagged_entries: Final = tuple(chain.from_iterable(registry.values())) + return tuple(tagged.strategy for tagged in tagged_entries) + + +def _find_program( + routers: tuple["OracleRouter", ...], program_id: str, user_api_key_dict: UserAPIKeyAuth +) -> tuple["OracleRouter", "ProgramBinding"] | None: + """The caller's program with this id. An admin may also reach another key's program of that id.""" + for candidate in routers: + owned = candidate.binding(program_id, user_api_key_dict.api_key) + if owned is not None: + return candidate, owned + if not _is_admin(user_api_key_dict): + return None + for candidate in routers: + for any_owner in candidate.bindings_named(program_id): + return candidate, any_owner + return None + + +@router.post( + "/oracle_router/feedback", + tags=["oracle_router"], + dependencies=[Depends(user_api_key_auth)], + response_model=OracleRouterFeedbackResponse, +) +async def submit_oracle_router_feedback( + data: OracleRouterFeedbackRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> OracleRouterFeedbackResponse: + """Complete a program: its model slot is released now, its verifier runs in the background. + + Programs are private to the key that started them; an admin may complete any key's program. 404 when + the caller holds no program with that id, which also covers programs completed earlier. A program none + of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed + response completes it with this feedback. + """ + routers: Final = _oracle_routers() + if not routers: + raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."}) + found: Final = _find_program(routers, data.program_id, user_api_key_dict) + if found is None: + raise HTTPException( + status_code=404, detail={"error": f"Unknown or already completed program {data.program_id!r}."} + ) + holder, binding = found + try: + holder.complete( + data.program_id, score=data.score, cost=data.cost, payload=data.payload, owner=binding.api_key_hash + ) + except VerificationBacklogFull as error: + raise HTTPException(status_code=503, detail={"error": str(error)}) from error + return OracleRouterFeedbackResponse( + program_id=data.program_id, + router_name=holder.router_name, + model=binding.model, + pending_verifications=holder.pending_verifications, + deferred=binding.requests == 0, + ) + + +@router.get( + "/oracle_router/state", + tags=["oracle_router"], + dependencies=[Depends(user_api_key_auth)], + response_model=OracleRouterStateResponse, +) +async def get_oracle_router_state( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> OracleRouterStateResponse: + """Decision-maker state, active bindings and recent verified feedback for every ORACLE router. Admin only.""" + if not _is_admin(user_api_key_dict): + raise HTTPException(status_code=403, detail={"error": CommonProxyErrors.not_allowed_access.value}) + routers: Final = _oracle_routers() + if not routers: + raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."}) + return OracleRouterStateResponse(routers=[await candidate.get_state_snapshot() for candidate in routers]) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b64c88521dc..74bfed5a41f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -803,6 +803,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( # n from litellm.proxy.management_endpoints.model_management_endpoints import ( router as model_management_router, ) +from litellm.proxy.management_endpoints.oracle_router_endpoints import ( + router as oracle_router_management_router, +) from litellm.proxy.management_endpoints.organization_endpoints import ( router as organization_router, ) @@ -22370,6 +22373,7 @@ app.include_router(budget_management_router) app.include_router(model_management_router) app.include_router(model_access_group_management_router) app.include_router(auto_router_management_router) +app.include_router(oracle_router_management_router) app.include_router(tag_management_router) app.include_router(workflow_management_router) app.include_router(memory_router) diff --git a/litellm/router_api.py b/litellm/router_api.py index f22e9b1f351..27973ba41e2 100644 --- a/litellm/router_api.py +++ b/litellm/router_api.py @@ -62,6 +62,7 @@ if TYPE_CHECKING: from litellm.router_strategy.auto_router.auto_router import AutoRouter from litellm.router_strategy.budget_limiter import RouterBudgetLimiting from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter + from litellm.router_strategy.oracle_router.oracle_router import OracleRouter from litellm.router_strategy.quality_router.quality_router import QualityRouter from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets as _AttemptedFallbackTargets @@ -104,6 +105,7 @@ if TYPE_CHECKING: | dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] | dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] | dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] + | dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]] | list[RoutingPlugin] | RouterBudgetLimiting | DualCache @@ -166,6 +168,7 @@ PluginKind: TypeAlias = Literal[ "complexity", "quality", "adaptive", + "oracle", "routing", "budget_limiter", "prompt_caching_pins", @@ -1001,6 +1004,11 @@ class RouterRegistry: self, kind: Literal["adaptive"] ) -> "dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]]": ... # mutable-ok: the live dict + @overload + def plugins( + self, kind: Literal["oracle"] + ) -> "dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]]": ... # mutable-ok: the live dict + @overload def plugins(self, kind: Literal["routing"]) -> "list[RoutingPlugin]": ... # mutable-ok: the live list @@ -1012,7 +1020,7 @@ class RouterRegistry: def plugins(self, kind: PluginKind) -> "RegistryPlugins": """ - Plugins configured by the deployments: ``auto``, ``complexity``, ``quality`` and ``adaptive`` + Plugins configured by the deployments: ``auto``, ``complexity``, ``quality``, ``adaptive`` and ``oracle`` sub-routers by deployment name, ``routing`` plugins, the deployment ``budget_limiter`` and the ``prompt_caching_pins`` store (were ``auto_routers``, ``complexity_routers``, ``quality_routers``, ``adaptive_routers``, ``routing_plugins``, @@ -1027,6 +1035,8 @@ class RouterRegistry: return r._core_quality_routers if kind == "adaptive": return r._core_adaptive_routers + if kind == "oracle": + return r._core_oracle_routers if kind == "routing": return r._core_routing_plugins if kind == "budget_limiter": diff --git a/litellm/router_python_core.py b/litellm/router_python_core.py index 4b5ecc82581..1b375285a56 100644 --- a/litellm/router_python_core.py +++ b/litellm/router_python_core.py @@ -448,6 +448,7 @@ from litellm.types.llms.openai import ( from litellm.types.router import ( CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, VALID_LITELLM_ENVIRONMENTS, + AdaptiveRouterPreferences, AlertingConfig, AllowedFailsPolicy, AssistantsTypedDict, @@ -552,6 +553,7 @@ if TYPE_CHECKING: from litellm.router_strategy.complexity_router.complexity_router import ( ComplexityRouter, ) + from litellm.router_strategy.oracle_router.oracle_router import OracleRouter from litellm.router_strategy.quality_router.quality_router import ( QualityRouter, ) @@ -573,6 +575,7 @@ else: ComplexityRouter = Any AdaptiveRouter = Any QualityRouter = Any + OracleRouter = Any # rebind-ok: runtime stand-in for the TYPE_CHECKING-only import above PreRoutingHookResponse = Any RouterStrategySelector: TypeAlias = ( @@ -1812,6 +1815,7 @@ class PythonCore(DeprecatedRouterMembers): self._core_complexity_routers: dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] = {} self._core_adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] = {} self._core_quality_routers: dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] = {} + self._core_oracle_routers: dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]] = {} # mutable-ok: registry self._core_routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else [] # Initialize model_group_alias early since it's used in set_model_list @@ -2111,6 +2115,7 @@ class PythonCore(DeprecatedRouterMembers): _routers_holding_global_callbacks.discard(self) slack_alerting_loggers: Final = self._slack_alerting_loggers() adaptive_router_hooks: Final = self._adaptive_router_hooks() + oracle_router_hooks: Final = self._oracle_router_hooks() vacated_keys: Final = self._host.callbacks.drop_callbacks( ( *self._callback_selectors(), @@ -2118,6 +2123,7 @@ class PythonCore(DeprecatedRouterMembers): *(self.optional_callbacks or ()), *slack_alerting_loggers, *adaptive_router_hooks, + *oracle_router_hooks, ), method_owners=(self, *slack_alerting_loggers), ) @@ -2130,9 +2136,17 @@ class PythonCore(DeprecatedRouterMembers): adaptive_survivor: Final = next((router for router in survivors if router._core_adaptive_routers), None) if adaptive_survivor is not None and vacated_keys & adaptive_keys: adaptive_survivor._sync_adaptive_router_hooks() + oracle_keys: Final = frozenset(map(self._host.callbacks.get_custom_logger_key, oracle_router_hooks)) + oracle_survivor: Final = next((router for router in survivors if router._core_oracle_routers), None) + if oracle_survivor is not None and vacated_keys & oracle_keys: + oracle_survivor._sync_oracle_router_hooks() def _core_serve_through_request_logging(self) -> None: - self._request_scoped_selectors = (*self._callback_selectors(), *self._request_adaptive_router_hooks()) + self._request_scoped_selectors = ( + *self._callback_selectors(), + *self._request_adaptive_router_hooks(), + *self._request_oracle_router_hooks(), + ) self._request_scoped_policies = (*self._optional_loggers(), *self._slack_alerting_loggers()) self._release_global_callbacks() @@ -2170,6 +2184,27 @@ class PythonCore(DeprecatedRouterMembers): for tagged in chain.from_iterable(self._core_adaptive_routers.values()) ) + def _request_oracle_router_hooks(self) -> tuple[CustomLogger, ...]: + from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook + + return tuple( + OracleRouterPostCallHook(oracle_router=tagged.strategy) + for tagged in chain.from_iterable(self._core_oracle_routers.values()) + ) + + def _oracle_router_hooks(self) -> tuple[CustomLogger, ...]: + from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook + + own_strategy_ids: Final = frozenset( + id(tagged.strategy) for tagged in chain.from_iterable(self._core_oracle_routers.values()) + ) + hooks: Final = self._host.callbacks.get_custom_loggers_for_type(OracleRouterPostCallHook) + return tuple( + hook + for hook in hooks + if isinstance(hook, OracleRouterPostCallHook) and id(hook.oracle_router) in own_strategy_ids + ) + def _routing_callbacks(self) -> Iterable[object]: if not (self._request_scoped_selectors or self._request_scoped_policies): return self._host.callback_list("callbacks") @@ -11543,6 +11578,8 @@ class PythonCore(DeprecatedRouterMembers): self._unregister_pre_routing_strategy(registry, model_name, tags) if self._unregister_pre_routing_strategy(self._core_adaptive_routers, model_name, tags): self._sync_adaptive_router_hooks() + if self._unregister_pre_routing_strategy(self._core_oracle_routers, model_name, tags): + self._sync_oracle_router_hooks() def _finalize_adaptive_router_if_configured(self) -> None: """Locate every adaptive-router deployment in the finalized model_list and @@ -11675,6 +11712,107 @@ class PythonCore(DeprecatedRouterMembers): len(config.available_models), ) + def _is_oracle_router_deployment(self, litellm_params: LiteLLM_Params) -> bool: + return classify_strategy_router_model(litellm_params.model) == "oracle" + + def _finalize_oracle_router_if_configured(self) -> None: + """Deferred like the adaptive router, since a `pre_routing` decision maker wraps a strategy router that + may appear later in the list. Idempotent across hot reloads, so a router that already holds program + bindings and learned state is never rebuilt.""" + for model_id in self._core_get_model_ids(): + deployment = self._core_get_deployment(model_id=model_id) + if deployment is None or not self._is_oracle_router_deployment(litellm_params=deployment.litellm_params): + continue + if self._has_registered_strategy( + self._core_oracle_routers, deployment.model_name, self._deployment_tags(deployment) + ): + continue + self.init_oracle_router_deployment(deployment=deployment) + self._sync_oracle_router_hooks() + + def _sync_oracle_router_hooks(self) -> None: + """Exactly one post-call hook per registered oracle router.""" + from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook + + for callback in self._host.callbacks.get_custom_loggers_for_type(OracleRouterPostCallHook): + self._host.callbacks.remove_callback_from_all_lists(callback) + for tagged in chain.from_iterable(self._core_oracle_routers.values()): + self._host.callbacks.add_litellm_callback(OracleRouterPostCallHook(oracle_router=tagged.strategy)) + + def _resolve_strategy_router(self, model_name: str) -> "PreRoutingStrategy | None": + """The first strategy router registered under `model_name`, for ORACLE's `pre_routing` decision maker.""" + for registry in ( + self._core_complexity_routers, + self._core_quality_routers, + self._core_adaptive_routers, + self._core_auto_routers, + ): + tagged_routers = registry.get(model_name) + if tagged_routers: + return tagged_routers[0].strategy + return None + + def _deployment_input_costs(self, model_names: Sequence[str]) -> Mapping[str, float]: + """``input_cost_per_token`` of the first deployment of each model name; litellm_params wins over model_info.""" + named: Final = ( + (name, self._core_get_deployment_by_model_group_name(model_group_name=name)) for name in model_names + ) + return MappingProxyType( + { + name: deployment.model_info.input_cost_per_token + for name, deployment in named + if deployment is not None and deployment.model_info.input_cost_per_token is not None + } + ) + + def _declared_adaptive_prefs(self, model_name: str) -> AdaptiveRouterPreferences | None: + deployment: Final = self._core_get_deployment_by_model_group_name(model_group_name=model_name) + extra: Final = deployment.model_info.model_extra if deployment is not None else None + raw: Final = extra.get("adaptive_router_preferences") if extra else None + return None if raw is None else AdaptiveRouterPreferences.model_validate(raw) + + def _deployment_adaptive_prefs(self, model_names: Sequence[str]) -> Mapping[str, AdaptiveRouterPreferences]: + """``model_info.adaptive_router_preferences`` of the first deployment of each model name, where declared.""" + declared: Final = ((name, self._declared_adaptive_prefs(name)) for name in model_names) + return MappingProxyType({name: prefs for name, prefs in declared if prefs is not None}) + + def init_oracle_router_deployment(self, deployment: Deployment) -> None: + """Build an OracleRouter from `oracle_router_config` and register it under `deployment.model_name`.""" + from litellm.router_strategy.oracle_router.decision import build_decision_maker + from litellm.router_strategy.oracle_router.oracle_router import OracleRouter + from litellm.router_strategy.oracle_router.verifier import build_verifier + from litellm.types.router import OracleRouterConfig + + raw_config: Final = deployment.litellm_params.oracle_router_config + if raw_config is None: + raise ValueError("oracle_router_config is required for oracle-router deployments.") + config: Final = OracleRouterConfig.model_validate(raw_config) + if deployment.model_info.db_model and "custom" in (config.decision_maker.type, config.verifier.type): + # A custom decision maker or verifier imports and runs Python named by the deployment. Only the + # operator's config file may name it; a model written through the API (any role) must not. + raise ValueError( + f"oracle_router deployment {deployment.model_name!r}: custom decision makers and verifiers " + "can only be configured in the proxy config file, not through the model API." + ) + oracle_router: Final = OracleRouter( + router_name=deployment.model_name, + config=config, + decision_maker=build_decision_maker( + config.decision_maker, + tuple(config.available_models), + self._deployment_input_costs(config.available_models), + self._resolve_strategy_router, + self._deployment_adaptive_prefs(config.available_models), + ), + verifier=build_verifier(config.verifier), + ) + self._register_pre_routing_strategy( + registry=self._core_oracle_routers, + deployment=deployment, + strategy=oracle_router, + strategy_label="Oracle-router", + ) + def _is_quality_router_deployment(self, litellm_params: LiteLLM_Params) -> bool: """ Check if the deployment is a quality-router deployment. @@ -11829,6 +11967,7 @@ class PythonCore(DeprecatedRouterMembers): # Deferred: build the AdaptiveRouter strategy now that all underlying # deployments have been registered. self._finalize_adaptive_router_if_configured() + self._finalize_oracle_router_if_configured() def _add_deployment(self, deployment: Deployment) -> Deployment: import os @@ -12214,6 +12353,8 @@ class PythonCore(DeprecatedRouterMembers): ) ): self._finalize_adaptive_router_if_configured() + if self._is_oracle_router_deployment(litellm_params=deployment.litellm_params): + self._finalize_oracle_router_if_configured() return deployment except Exception as e: if self.ignore_invalid_deployments: @@ -16672,6 +16813,7 @@ class PythonCore(DeprecatedRouterMembers): self._core_complexity_routers, self._core_adaptive_routers, self._core_quality_routers, + self._core_oracle_routers, ) if not any(registries): return None @@ -16773,6 +16915,7 @@ class PythonCore(DeprecatedRouterMembers): self._core_complexity_routers, self._core_adaptive_routers, self._core_quality_routers, + self._core_oracle_routers, ) ): return registered_model_name diff --git a/litellm/router_strategy/auto_router/auto_router_model_naming.py b/litellm/router_strategy/auto_router/auto_router_model_naming.py index f5aeccd9305..cb55f7e05fd 100644 --- a/litellm/router_strategy/auto_router/auto_router_model_naming.py +++ b/litellm/router_strategy/auto_router/auto_router_model_naming.py @@ -26,7 +26,7 @@ from litellm.router_strategy.complexity_router.config import ( AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/" -StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"] +StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality", "oracle"] StrategyRouterDependencyRole: TypeAlias = Literal[ "tier", "default", "classifier", "embedding", "evaluation", "compactor" @@ -53,6 +53,7 @@ STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset( "adaptive_router_config", "quality_router_config", "quality_router_default_model", + "oracle_router_config", } ) @@ -65,6 +66,7 @@ _REQUIRED_FIELD_GROUPS: Final[Mapping[StrategyRouterKind, tuple[tuple[str, ...], "complexity": (("complexity_router_config", "complexity_router_default_model"),), "adaptive": (("adaptive_router_config",),), "quality": (("quality_router_config", "quality_router_default_model"),), + "oracle": (("oracle_router_config",),), } @@ -84,6 +86,8 @@ def classify_strategy_router_model(model: str) -> StrategyRouterKind | None: return "adaptive" if remainder.startswith("quality_router"): return "quality" + if remainder.startswith("oracle_router"): + return "oracle" return "semantic" @@ -143,6 +147,8 @@ def strategy_router_dependencies( ) if kind == "adaptive": return _pool(_mapping(litellm_params.get("adaptive_router_config")).get("available_models"), "tier") + if kind == "oracle": + return _pool(_mapping(litellm_params.get("oracle_router_config")).get("available_models"), "tier") if kind == "quality": quality: Final = _mapping(litellm_params.get("quality_router_config")) return tuple( diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md new file mode 100644 index 00000000000..44d4200a164 --- /dev/null +++ b/litellm/router_strategy/oracle_router/README.md @@ -0,0 +1,101 @@ +# ORACLE router + +Program-level routing for agentic workloads: an agentic task (a *program*) is bound to one model at its +first request, every later request with the same `program_id` reuses that binding, and the model is +chosen by a pluggable decision maker that learns from a pluggable verifier's delayed verdict. +Paper: https://arxiv.org/abs/2607.22465 + +## Config + +```yaml +model_list: + - model_name: oracle-router + litellm_params: + model: auto_router/oracle_router + oracle_router_config: + available_models: ["smart", "fast"] + decision_maker: { type: thompson, weights: { quality: 0.7, cost: 0.3 } } + verifier: { type: guardrail, guardrail: answer-judge } +``` + +Clients send `metadata.program_id` (or `litellm_session_id`, or the `x-litellm-program-id` header) +with every request of a task. Requests without any id are routed statelessly and never learn. Concurrent +first requests of one program wait for the decision in flight, so a program is bound exactly once. + +## Decision makers + +| `type` | binds a program by | learns | +|---|---|---| +| `thompson` (default) | LiteLLM's adaptive-router bandit: a Beta posterior per (request type, model), Thompson sampling, cost-weighted pick with `weights: {quality, cost}` | from verified program scores | +| `pre_routing` | the strategy router named by `router` (complexity, quality, adaptive or semantic), once per program | no | +| `fixed` | always `model` | no | +| `custom` | `path: pkg.mod:Factory`, called with the model tuple; must expose `models`, `async select`, `update`, `snapshot` | up to you | + +## Verifiers + +| `type` | scores a finished program by | +|---|---| +| `guardrail` | applying the LiteLLM guardrail named `guardrail` to the program's final turn, with the transcript as `structured_messages` and the feedback `payload` as `request_data["metadata"]`: pass = 1, blocked = 0 (read from the status the guardrail records, so an `llm_as_a_judge` with `on_failure: log` still scores 0 when the answer fails). A guardrail that could not evaluate the program, for example a judge whose model was unreachable, counts as a failed verification and teaches the decision maker nothing | +| `reported` (default) | `metadata.program_score` on the last request, or `POST /oracle_router/feedback` | +| `custom` | `path: pkg.mod:Factory`, called with no arguments; must expose `async verify(outcome)` | + +A verified score `s` adds `s` successes and `1 - s` failures to the cell of the model that served the +program. Updates are applied in verification-arrival order. A verifier that raises, or returns a score +that is not a finite number, counts as a failed verification (`verifications_failed` in the state) and +never reaches the decision maker. + +## Finishing a program + +Either mark the last request with `metadata.program_done: true` (plus `program_score` for the reported +verifier), or call `POST /oracle_router/feedback {"program_id": ..., "score": ..., "cost": ...}` from +the harness once the task is graded. Feedback that arrives before any response of the program was +observed (it raced the last response's logging, or every request failed) is deferred: the reply says +`deferred: true` and the next observed response completes the program with it, so nothing is ever +verified or learned without an answer. A guardrail verifier also fails the verification of a program +whose final response has no text, rather than reading a judge's skip as a pass. `GET /oracle_router/state` +(admin) shows bindings, decision-maker state and recent feedback. The response header +`x-litellm-oracle-router-model` names the bound model. + +Verification runs in the background and is bounded: at most `max_pending_verifications` (default 64) +verifications are in flight per router, and each one is given `verification_timeout_seconds` (default +120) before it counts as failed. While the backlog is full a completion is refused and the program stays +bound, so it can be completed again once the backlog drains: the feedback endpoint answers 503 and a +`program_done` request records the refusal in `last_failure`; `verifications_rejected` in the state +counts them. + +## Security + +A program is private to the API key that started it: the same `program_id` under another key is another +program, so no caller can ride on, observe or complete someone else's binding. `POST /oracle_router/feedback` +completes the caller's own program; an admin key may complete any key's. The post-call hook only reads +requests the Router stamped as routed by this router (`routing_decision`), so ORACLE keys placed in the +metadata of a request to some other model are ignored, and the router drops any caller-supplied copy of +its internal metadata keys before stamping its own. + +A `pre_routing` decision maker hands the strategy router it wraps what LiteLLM forwards to any detached +sub-call: the caller's identity (`user_api_key`, its hash and alias, team, user, organization and end user, +and the auth object the Router filters deployments by, minus the request's budget reservation), nothing else +of the request, and the program's `turn_off_message_logging` opt-out. A classifier call made on the program's +behalf is therefore billed to the caller's key and team and never logs a prompt the caller marked private; +the identity is used for that call only and is not kept on the program. Verification runs under the +program's opt-out as well: a program is private once any of its requests opted out, whether or not that +request produced an answer, so a `guardrail` verifier's judge call logs neither the prompt nor the +transcript. LiteLLM redacts a private request's messages and answer before the router's hook sees them, so +a `guardrail` verifier cannot judge a program whose final turn was redacted: that is a failed verification +and teaches the decision maker nothing. Use the `reported` verifier for routers whose programs opt out of +logging. A `guardrail` verifier's judge call carries no caller identity, as the proxy's own +`llm_as_a_judge` flow does not, so its spend is not attributed to the program's key or team. The feedback +`payload` reaches the guardrail with the proxy-owned keys removed (`user_api_key_*`, the sub-call origin, +headers and the routing decision), so a caller cannot pose as the proxy to a guardrail that forwards +identity to its vendor. + +`custom` decision makers and verifiers import Python named by the deployment, so they are honoured only +for deployments in the proxy config file. A model written through the API (`/model/new`, `/model/update`, +whatever the caller's role) with a `custom` type is rejected when the router is built and never imported. + +## Limits + +State is in memory per proxy process; it is not persisted or shared across replicas. The paper's +dispatch scheduler (DISC) is not part of this strategy. The router and its hook write nothing to the +process log: `GET /oracle_router/state` carries the counters and the last background failure +(`last_failure`) instead. diff --git a/litellm/router_strategy/oracle_router/__init__.py b/litellm/router_strategy/oracle_router/__init__.py new file mode 100644 index 00000000000..d5bc771d923 --- /dev/null +++ b/litellm/router_strategy/oracle_router/__init__.py @@ -0,0 +1,32 @@ +from litellm.router_strategy.oracle_router.decision import ( + DecisionMaker, + FixedDecisionMaker, + PreRoutingDecisionMaker, + ProgramContext, + ThompsonDecisionMaker, + build_decision_maker, +) +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, ProgramBinding +from litellm.router_strategy.oracle_router.verifier import ( + GuardrailVerifier, + ProgramOutcome, + ReportedVerifier, + Verifier, + build_verifier, +) + +__all__ = [ + "DecisionMaker", + "FixedDecisionMaker", + "GuardrailVerifier", + "OracleRouter", + "PreRoutingDecisionMaker", + "ProgramBinding", + "ProgramContext", + "ProgramOutcome", + "ReportedVerifier", + "ThompsonDecisionMaker", + "Verifier", + "build_decision_maker", + "build_verifier", +] diff --git a/litellm/router_strategy/oracle_router/config.py b/litellm/router_strategy/oracle_router/config.py new file mode 100644 index 00000000000..a7ed8ccc1e7 --- /dev/null +++ b/litellm/router_strategy/oracle_router/config.py @@ -0,0 +1,18 @@ +"""Metadata keys and headers shared by the ORACLE router, its post-call hook and the proxy endpoints.""" + +from typing import Final + +ORACLE_ROUTER_MODEL_PREFIX: Final[str] = "auto_router/oracle_router" + +CHOSEN_MODEL_METADATA_KEY: Final[str] = "oracle_router_chosen_model" +PROGRAM_ID_METADATA_KEY: Final[str] = "oracle_router_program_id" +RESPONSE_HEADER: Final[str] = "x-litellm-oracle-router-model" +PROGRAM_ID_HEADER: Final[str] = "x-litellm-program-id" + +PROGRAM_DONE_KEY: Final[str] = "program_done" +PROGRAM_SCORE_KEY: Final[str] = "program_score" + +SESSION_ID_FALLBACK_KEYS: Final[tuple[str, ...]] = ("litellm_session_id", "session_id") + +FEEDBACK_HISTORY_SIZE: Final[int] = 200 +PROGRAM_SWEEP_THRESHOLD: Final[int] = 1024 diff --git a/litellm/router_strategy/oracle_router/decision.py b/litellm/router_strategy/oracle_router/decision.py new file mode 100644 index 00000000000..d31f1438a5a --- /dev/null +++ b/litellm/router_strategy/oracle_router/decision.py @@ -0,0 +1,248 @@ +"""Decision makers: the pluggable policy that binds a program to a model. + +``DecisionMaker`` is the whole contract. The default, ``thompson``, is LiteLLM's own adaptive-router +bandit (a Beta posterior per request type and model, Thompson sampling, cost-weighted pick) trained on +verified program outcomes instead of per-turn regex signals. ``pre_routing`` lets any LiteLLM strategy +router (complexity, quality, adaptive, semantic) make the proposal once per program, ``fixed`` pins one +model, and ``custom`` imports an operator's class. +""" + +import importlib +import itertools +import random +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final, Protocol, runtime_checkable + +from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy +from litellm.router_strategy.adaptive_router.bandit import BanditCell, apply_delta, initial_cell, pick_best +from litellm.types.router import ( + AdaptiveRouterPreferences, + AdaptiveRouterWeights, + OracleDecisionMakerConfig, + PreRoutingStrategy, + RequestType, +) + +_NO_CALLER: Final[Mapping[str, object]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class ProgramContext: + """What a decision maker sees when a program arrives: its initial request and LiteLLM's request type.""" + + program_id: str + prompt: str + request_type: RequestType + message_logging_off: bool = False + caller: Mapping[str, object] = field(default_factory=lambda: _NO_CALLER) + """The caller's identity metadata, so a sub-call made on the program's behalf is billed to the caller.""" + + +@runtime_checkable +class DecisionMaker(Protocol): + @property + def models(self) -> tuple[str, ...]: ... + + async def select(self, context: ProgramContext) -> str: ... + + def update(self, context: ProgramContext, model: str, score: float) -> None: ... + + def snapshot(self) -> Mapping[str, object]: ... + + +def _check_model(models: Sequence[str], model: str) -> None: + if model not in models: + raise KeyError(f"unknown model {model!r}; configured models: {list(models)}") + + +_UNIFORM_PRIOR: Final = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) + + +class ThompsonDecisionMaker: + """LiteLLM's adaptive-router bandit, keyed by (request type, model) and updated with verified scores. + + Cells start from the same priors the adaptive router uses, read from each model's + ``model_info.adaptive_router_preferences`` (tier 2 when a model declares none). A verified score ``s`` + in [0, 1] adds ``s`` successes and ``1 - s`` failures to the cell of the model that served the program. + Selection samples every cell once and picks the best quality/cost score with the adaptive router's + weights, so ``weights.cost`` is the only cost knob. + """ + + def __init__( + self, + models: Sequence[str], + model_costs: Mapping[str, float], + weights: AdaptiveRouterWeights | None = None, + model_prefs: Mapping[str, AdaptiveRouterPreferences] | None = None, + seed: int = 0, + ) -> None: + if not models: + raise ValueError("a decision maker needs at least one model") + self._models: Final[tuple[str, ...]] = tuple(models) + costs: Final = {m: float(model_costs.get(m, 0.0)) for m in self._models} + self._costs: Final[dict[str, float]] = costs # mutable-ok: pick_best takes a dict + self._weights: Final[AdaptiveRouterWeights] = weights or AdaptiveRouterWeights() + self._rng: Final = random.Random(seed) + prefs: Final = model_prefs or {} + grid: Final = tuple(itertools.product(RequestType, self._models)) + cells: Final = {(rt, model): initial_cell(prefs.get(model, _UNIFORM_PRIOR), rt) for rt, model in grid} + self._cells: Final[dict[tuple[RequestType, str], BanditCell]] = cells # mutable-ok: replaced on update + self.pulls: Final[dict[str, int]] = {m: 0 for m in self._models} # mutable-ok: counters + + @property + def models(self) -> tuple[str, ...]: + return self._models + + async def select(self, context: ProgramContext) -> str: + cells: Final = {model: self._cells[(context.request_type, model)] for model in self._models} + chosen: Final = pick_best(cells, self._costs, self._weights.quality, self._weights.cost, rng=self._rng) + self.pulls[chosen] += 1 + return chosen + + def update(self, context: ProgramContext, model: str, score: float) -> None: + _check_model(self._models, model) + key: Final = (context.request_type, model) + self._cells[key] = apply_delta(self._cells[key], score, 1.0 - score) + + def estimate(self, context: ProgramContext, model: str) -> float: + _check_model(self._models, model) + return self._cells[(context.request_type, model)].mean + + def snapshot(self) -> Mapping[str, object]: + cells: Final = { + f"{request_type.value}/{model}": {"mean": round(cell.mean, 4), "samples": cell.total_samples} + for (request_type, model), cell in self._cells.items() + if cell.total_samples > 0 + } + return { + "type": "thompson", + "weights": {"quality": self._weights.quality, "cost": self._weights.cost}, + "pulls": dict(self.pulls), + "cells": cells, + } + + +class FixedDecisionMaker: + """Always the same model; the single-model baseline that still gets verified and reported.""" + + def __init__(self, models: Sequence[str], model: str) -> None: + _check_model(models, model) + self._models: Final[tuple[str, ...]] = tuple(models) + self._model: Final[str] = model + self._updates: int = 0 + + @property + def models(self) -> tuple[str, ...]: + return self._models + + async def select(self, context: ProgramContext) -> str: + return self._model + + def update(self, context: ProgramContext, model: str, score: float) -> None: + _check_model(self._models, model) + self._updates += 1 + + def snapshot(self) -> Mapping[str, object]: + return {"type": "fixed", "model": self._model, "updates": self._updates} + + +def strategy_request_kwargs(context: ProgramContext) -> dict[str, object]: # mutable-ok: pre-routing contract + """The request a wrapped strategy router sees for the program: the caller's identity, so a classifier call + the strategy makes is billed to the caller's key and team, and the program's message-logging opt-out where + LiteLLM reads it. A fresh dict per call: the strategies write their decision into the bucket they are handed.""" + privacy: Final[Mapping[str, object]] = {"turn_off_message_logging": True} if context.message_logging_off else {} + return {"metadata": {**context.caller, **privacy}} + + +class PreRoutingDecisionMaker: + """Let one of LiteLLM's own strategy routers propose the model, once per program. + + The wrapped router (complexity, quality, adaptive or semantic) sees the program's initial request as a + single user message. Its per-request decision becomes a per-program binding, and the program's verified + outcome is recorded against that model, so the state endpoint shows what the strategy actually achieved. + """ + + def __init__(self, models: Sequence[str], router_name: str, strategy: PreRoutingStrategy) -> None: + if not models: + raise ValueError("a decision maker needs at least one model") + self._models: Final[tuple[str, ...]] = tuple(models) + self._router_name: Final[str] = router_name + self._strategy: Final[PreRoutingStrategy] = strategy + self._proposals: Final[dict[str, int]] = {} # mutable-ok: proposal counts per model, incremented per program + self._fallbacks: int = 0 + self._updates: int = 0 + + @property + def models(self) -> tuple[str, ...]: + return self._models + + async def select(self, context: ProgramContext) -> str: + with inherit_message_logging_privacy(context.message_logging_off): + response: Final = await self._strategy.async_pre_routing_hook( + model=self._router_name, + request_kwargs=strategy_request_kwargs(context), + messages=[{"role": "user", "content": context.prompt}], + ) + proposed: Final = response.model if response is not None and response.model in self._models else None + if proposed is None: + self._fallbacks += 1 + chosen: Final = proposed if proposed is not None else self._models[0] + self._proposals[chosen] = self._proposals.get(chosen, 0) + 1 + return chosen + + def update(self, context: ProgramContext, model: str, score: float) -> None: + _check_model(self._models, model) + self._updates += 1 + + def snapshot(self) -> Mapping[str, object]: + return { + "type": "pre_routing", + "router": self._router_name, + "proposals": dict(self._proposals), + "fallbacks": self._fallbacks, + "updates": self._updates, + } + + +def load_object(path: str) -> object: + """Import ``package.module:attribute``.""" + module_name, separator, attribute = path.partition(":") + if not separator or not module_name or not attribute: + raise ValueError(f"expected 'package.module:Attribute', got {path!r}") + loaded: Final[object] = getattr(importlib.import_module(module_name), attribute) # pyright: ignore[reportAny] # module attributes are untyped + return loaded + + +def build_decision_maker( + config: OracleDecisionMakerConfig, + models: Sequence[str], + model_costs: Mapping[str, float], + resolve_strategy: Callable[[str], PreRoutingStrategy | None], + model_prefs: Mapping[str, AdaptiveRouterPreferences] | None = None, +) -> DecisionMaker: + """Instantiate the configured decision maker over ``models`` (strongest first).""" + match config.type: + case "thompson": + return ThompsonDecisionMaker( + models, model_costs, weights=config.weights, model_prefs=model_prefs, seed=config.seed + ) + case "fixed": + return FixedDecisionMaker(models, model=config.model or models[0]) + case "pre_routing": + router_name: Final = config.router or "" + strategy: Final = resolve_strategy(router_name) + if strategy is None: + raise ValueError( + f"oracle_router decision_maker.router {router_name!r} is not a configured strategy router" + ) + return PreRoutingDecisionMaker(models, router_name=router_name, strategy=strategy) + case "custom": + factory: Final = load_object(config.path or "") + if not callable(factory): + raise TypeError(f"oracle_router decision_maker.path {config.path!r} is not callable") + instance: Final = factory(tuple(models)) + if not isinstance(instance, DecisionMaker): + raise TypeError(f"{config.path!r} must build an object with select, update, snapshot and models") + return instance diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py new file mode 100644 index 00000000000..3d2f2ffc3c3 --- /dev/null +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -0,0 +1,119 @@ +"""Post-call hook for the ORACLE router. + +After every completion it adds LiteLLM's computed spend and the latest answer to the program's row, and +when the request carried ``metadata.program_done`` it completes the program: the slot is released at once +and the verifier runs in the background. Only requests the Router stamped as routed by this ORACLE router +are read, and only against the program of the key that sent them. Nothing here can fail a request, and +nothing here logs: a failure is kept on the router for ``/oracle_router/state``. +""" + +import math +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +from litellm.integrations.custom_logger import CustomLogger +from litellm.router_strategy.oracle_router.config import ( + CHOSEN_MODEL_METADATA_KEY, + PROGRAM_DONE_KEY, + PROGRAM_ID_METADATA_KEY, + PROGRAM_SCORE_KEY, + RESPONSE_HEADER, +) +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, as_str_mapping, logged_request_is_private +from litellm.router_strategy.oracle_router.verifier import response_text + +if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth + + +def _request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + """The request's metadata as the proxy saw it: the caller's bucket with the proxy-internal one on top.""" + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) + return _merged(litellm_params.get("metadata"), litellm_params.get("litellm_metadata")) + + +def _merged(metadata: object, litellm_metadata: object) -> Mapping[str, object]: + return {**as_str_mapping(metadata), **as_str_mapping(litellm_metadata)} + + +def _routed_by(metadata: Mapping[str, object], router_name: str) -> bool: + """True when the Router stamped this request as routed by this ORACLE router. + + The Router writes or clears ``routing_decision`` on every routing attempt, so a caller cannot make a + request to some other model look like one of this router's by putting ORACLE keys in its metadata. + """ + decision: Final = as_str_mapping(metadata.get("routing_decision")) + return decision.get("router_type") == "oracle" and decision.get("router_model_name") == router_name + + +def _owner(metadata: Mapping[str, object]) -> str | None: + """The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy.""" + value: Final = metadata.get("user_api_key_hash") + return str(value) if value else None + + +def _score(metadata: Mapping[str, object]) -> float | None: + raw: Final = metadata.get(PROGRAM_SCORE_KEY) + if isinstance(raw, bool) or not isinstance(raw, (int, float, str)): + return None + try: + score: Final = float(raw) + except ValueError: + return None + return score if math.isfinite(score) else None # a NaN would corrupt a bandit cell for good + + +class OracleRouterPostCallHook(CustomLogger): + """One hook per OracleRouter, registered in ``litellm.callbacks``.""" + + def __init__(self, oracle_router: OracleRouter) -> None: + self.oracle_router: Final[OracleRouter] = oracle_router + + async def async_post_call_response_headers_hook( + self, + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + response: object, + request_headers: dict[str, str] | None = None, # mutable-ok: CustomLogger's hook signature + litellm_call_info: dict[str, object] | None = None, # mutable-ok: CustomLogger's hook signature + ) -> dict[str, str] | None: # mutable-ok: CustomLogger's hook signature + metadata: Final = _merged(data.get("metadata"), data.get("litellm_metadata")) + if not _routed_by(metadata, self.oracle_router.router_name): + return None + chosen: Final = metadata.get(CHOSEN_MODEL_METADATA_KEY) + return {RESPONSE_HEADER: str(chosen)} if chosen else None + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs, response_obj, succeeded=True) + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs, response_obj, succeeded=False) + + def _record(self, kwargs: Mapping[str, object], response_obj: object, succeeded: bool) -> None: + try: + metadata: Final = _request_metadata(kwargs) + if not _routed_by(metadata, self.oracle_router.router_name): + return + program_id: Final = metadata.get(PROGRAM_ID_METADATA_KEY) + if not isinstance(program_id, str): + return + owner: Final = _owner(metadata) + if logged_request_is_private(kwargs): + self.oracle_router.mark_message_logging_off(program_id, owner) + if succeeded: + cost: Final = kwargs.get("response_cost") + self.oracle_router.observe_request( + program_id, + cost=float(cost) if isinstance(cost, (int, float)) else 0.0, + response_text=response_text(response_obj), + messages=kwargs.get("messages"), + owner=owner, + ) + if metadata.get(PROGRAM_DONE_KEY): + self.oracle_router.complete(program_id, score=_score(metadata), owner=owner) + except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes + self.oracle_router.note_failure("post-call hook", error) diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py new file mode 100644 index 00000000000..2df10409035 --- /dev/null +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -0,0 +1,548 @@ +"""The ORACLE router: one binding per agentic program, learned from verified, delayed feedback. + +A program is one agentic task made of many LLM requests that share a prefix. Its first request is bound +to a model by the decision maker and every later request with the same program id reuses that binding. +When the program finishes, the verifier scores it off the critical path, the score and LiteLLM's spend +for the program become one reward, and the decision maker learns from it in arrival order. + +Paper: ORACLE, https://arxiv.org/abs/2607.22465 +""" + +import asyncio +import time +from collections import deque +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, replace +from types import MappingProxyType +from typing import Final, NamedTuple + +from pydantic import TypeAdapter, ValidationError + +from litellm.litellm_core_utils.core_helpers import ( + get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict +) +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, + initialize_standard_callback_dynamic_params, +) +from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata +from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages +from litellm.litellm_core_utils.redact_messages import ( + should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # upstream helper takes a bare dict +) +from litellm.router_strategy.adaptive_router.classifier import classify_prompt +from litellm.router_strategy.oracle_router.config import ( + CHOSEN_MODEL_METADATA_KEY, + FEEDBACK_HISTORY_SIZE, + PROGRAM_ID_HEADER, + PROGRAM_ID_METADATA_KEY, + PROGRAM_SWEEP_THRESHOLD, + SESSION_ID_FALLBACK_KEYS, +) +from litellm.router_strategy.oracle_router.decision import DecisionMaker, ProgramContext +from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, Verifier, clamp_score +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import OracleRouterConfig, PreRoutingHookResponse, RequestType +from litellm.types.utils import ( + AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + RoutingDecisionCause, + StandardLoggingRoutingDecision, +) + +_LEARNING_DECISION_MAKERS: Final[frozenset[str]] = frozenset({"thompson"}) +_MESSAGES: Final = TypeAdapter(Sequence[AllMessageValues]) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECTS: Final = TypeAdapter(Sequence[object]) + + +def as_str_mapping(value: object) -> Mapping[str, object]: + """A typed, read-only view of a request-shaped mapping; empty for anything else.""" + if not isinstance(value, Mapping): + return _EMPTY + try: + return _STR_MAPPING.validate_python(value) + except ValidationError: + return _EMPTY + + +class ProgramKey(NamedTuple): + """A program is private to the API key that started it: the same id under another key is another program.""" + + owner: str | None + program_id: str + + +@dataclass(frozen=True, slots=True) +class DeferredCompletion: + """A completion that arrived before any response of the program was observed; the next one applies it.""" + + score: float | None + cost: float | None + payload: Mapping[str, object] + + +@dataclass(frozen=True, slots=True) +class ProgramBinding: + """One row of the lookup table: program id to model, plus what the proxy has seen of the program.""" + + program_id: str + model: str + context: ProgramContext + api_key_hash: str | None + bound_at: float + requests: int = 0 + cost: float = 0.0 + last_response_text: str = "" + last_messages: Sequence[AllMessageValues] = () + deferred: DeferredCompletion | None = None + message_logging_off: bool = False + """True once any request of the program opted out of message logging; verification then runs privately.""" + + +@dataclass(frozen=True, slots=True) +class FeedbackRecord: + program_id: str + model: str + request_type: str + score: float + cost: float + verified_at: float + + +def program_messages( + messages: list[dict[str, object]] | None, # mutable-ok: pre-routing contract + input: str | list[object] | None, # mutable-ok: pre-routing contract + request_kwargs: Mapping[str, object], +) -> Sequence[Mapping[str, object]] | None: + """The request's messages in chat shape whichever surface sent them: a Responses API ``input`` is converted + the way guardrails convert it, so a program started through ``/v1/responses`` is bound on its real prompt.""" + if messages or input is None: + return messages + return resolve_structured_messages( + None, + {**request_kwargs, "input": input, "litellm_metadata": as_str_mapping(request_kwargs.get("litellm_metadata"))}, + ) + + +def first_user_text(messages: Sequence[Mapping[str, object]] | None) -> str: + """The program's initial request: the first user message, text parts only.""" + for message in messages or (): + if message.get("role") != "user": + continue + content = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, Sequence): + parts = tuple(as_str_mapping(part) for part in _OBJECTS.validate_python(content)) + return "\n".join(str(part.get("text", "")) for part in parts if part.get("type") == "text") + return "" + return "" + + +def _metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + """The request's metadata as callers see it, with the proxy-internal bucket layered on top.""" + return {**as_str_mapping(request_kwargs.get("metadata")), **as_str_mapping(request_kwargs.get("litellm_metadata"))} + + +def _header(request_kwargs: Mapping[str, object], name: str) -> str | None: + headers: Final = as_str_mapping(request_kwargs.get("headers")) + matches: Final = tuple(value for key, value in headers.items() if key.lower() == name) + return str(matches[0]) if matches else None + + +def resolve_program_id(request_kwargs: Mapping[str, object], program_id_key: str) -> str | None: + """The program id of a request: ``metadata[program_id_key]``, then the session id spellings, then the header.""" + metadata: Final = _metadata(request_kwargs) + candidates: Final = ( + metadata.get(program_id_key), + *(metadata.get(key) for key in SESSION_ID_FALLBACK_KEYS), + request_kwargs.get("litellm_session_id"), + _header(request_kwargs, PROGRAM_ID_HEADER), + ) + return next((str(candidate) for candidate in candidates if candidate), None) + + +def _transcript(messages: object, fallback: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + if not messages: + return fallback + try: + return tuple(_MESSAGES.validate_python(messages)) + except ValidationError: + return fallback + + +def request_is_private(request_kwargs: Mapping[str, object]) -> bool: + """Whether LiteLLM will redact this request's messages when it logs them, by the logger's own rule: the + request's ``turn_off_message_logging``, its redaction headers, or the global setting.""" + kwargs: Final = dict(request_kwargs) + return _redacts( + { + "litellm_params": kwargs, + "standard_callback_dynamic_params": initialize_standard_callback_dynamic_params(kwargs), + } + ) + + +def logged_request_is_private(kwargs: Mapping[str, object]) -> bool: + """Whether LiteLLM redacted a logged request's messages, from the kwargs it hands a logging callback.""" + return _redacts( + { + **kwargs, + "standard_callback_dynamic_params": kwargs.get("standard_callback_dynamic_params") + or initialize_standard_callback_dynamic_params(dict(kwargs)), + } + ) + + +def _redacts(model_call_details: dict[str, object]) -> bool: # mutable-ok: the logger's own parameter shape + try: + return should_redact_message_logging(model_call_details) + except AttributeError: # malformed metadata makes the logger raise and log nothing, so the program stays private + return True + + +def forwardable_caller(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + """What a classifier call made on the program's behalf inherits from the request that bound it: the caller's + identity from whichever metadata bucket the surface used, with its budget reservation removed, stamped as the + sub-call it becomes. The routing decision, the logging payload and every other per-request state stay behind.""" + return MappingProxyType( + sanitized_forwardable_call_metadata(_metadata(request_kwargs), AUTOROUTER_CLASSIFIER_CALL_ORIGIN) + ) + + +def _api_key_hash(request_kwargs: Mapping[str, object]) -> str | None: + """The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy.""" + value: Final = _metadata(request_kwargs).get("user_api_key_hash") + return str(value) if value else None + + +def _stamp_internal( + request_kwargs: dict[str, object], # mutable-ok: pre-routing contract + key: str, + value: object | None, +) -> None: + """Write a router-internal metadata key where the proxy keeps its own state. + + Any copy the caller supplied is dropped from both buckets first, so the post-call hook only ever reads + what this router wrote; None clears the key. + """ + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")): + if isinstance(bucket, dict): + bucket.pop(key, None) # pyright: ignore[reportUnknownMemberType] # caller-supplied dict + if value is not None: + _, internal = get_or_create_metadata_bucket(request_kwargs) # pyright: ignore[reportUnknownVariableType] # bare dict upstream + internal[key] = value + + +class VerificationBacklogFull(RuntimeError): + """``max_pending_verifications`` verifications are in flight; the program stays bound for a later retry.""" + + +class OracleRouter: + """One instance per ``auto_router/oracle_router`` deployment.""" + + def __init__( + self, + router_name: str, + config: OracleRouterConfig, + decision_maker: DecisionMaker, + verifier: Verifier, + clock: Callable[[], float] = time.time, + ) -> None: + self.router_name: Final[str] = router_name + self.config: Final[OracleRouterConfig] = config + self.decision_maker: Final[DecisionMaker] = decision_maker + self.verifier: Final[Verifier] = verifier + self._clock: Final = clock + self._cause: Final[RoutingDecisionCause] = ( + "bandit" if config.decision_maker.type in _LEARNING_DECISION_MAKERS else "classifier_plugin" + ) + self._bindings: Final[dict[ProgramKey, ProgramBinding]] = {} # mutable-ok: the lookup table + self._binding_in_flight: Final[dict[ProgramKey, asyncio.Event]] = {} # mutable-ok: being bound right now + self._tasks: Final[set[asyncio.Task[float]]] = set() # mutable-ok: in-flight verifications + self._history: Final[deque[FeedbackRecord]] = deque(maxlen=FEEDBACK_HISTORY_SIZE) # mutable-ok: bounded log + self._update_lock: Final = asyncio.Lock() + self.programs_bound: int = 0 + self.programs_completed: int = 0 + self.programs_evicted: int = 0 + self.stateless_requests: int = 0 + self.verifications_failed: int = 0 + self.verifications_rejected: int = 0 + self.last_failure: str | None = None + + @property + def models(self) -> tuple[str, ...]: + return tuple(self.config.available_models) + + def note_failure(self, step: str, error: Exception) -> None: + """Keep the last failure of a background step for ``/oracle_router/state``. + + The router and its hook write nothing to the process log: every request input reaches them, and a + log call on that path makes CodeQL's log-injection analysis fall back to a whole-codebase pass that + no longer fits its limits on this codebase. The admin state endpoint carries the failure instead. + """ + self.last_failure = f"{step}: {type(error).__name__}: {error}" + + def context_for(self, program_id: str, prompt: str, request_kwargs: Mapping[str, object]) -> ProgramContext: + return ProgramContext( + program_id=program_id, + prompt=prompt, + request_type=classify_prompt(prompt), + message_logging_off=request_is_private(request_kwargs), + caller=forwardable_caller(request_kwargs), + ) + + def binding(self, program_id: str, owner: str | None = None) -> ProgramBinding | None: + """The program's row as seen by ``owner``, the key that started it (``user_api_key_hash``).""" + return self._bindings.get(ProgramKey(owner, program_id)) + + def bindings_named(self, program_id: str) -> tuple[ProgramBinding, ...]: + """Every key's program with this id, oldest first: an admin may complete another key's program.""" + matching: Final = (binding for key, binding in self._bindings.items() if key.program_id == program_id) + return tuple(sorted(matching, key=lambda binding: binding.bound_at)) + + def _evict_expired(self, now: float) -> None: + deadline: Final = now - self.config.program_ttl_seconds + expired: Final = tuple(key for key, binding in self._bindings.items() if binding.bound_at < deadline) + for key in expired: + self._bindings.pop(key, None) + overflow: Final = len(self._bindings) + 1 - self.config.max_programs # +1: room for the program being bound + oldest: Final = tuple(sorted(self._bindings, key=lambda key: self._bindings[key].bound_at))[: max(overflow, 0)] + for key in oldest: + self._bindings.pop(key, None) + self.programs_evicted += len(expired) + len(oldest) + + async def bind( + self, program_id: str, prompt: str, api_key_hash: str | None, request_kwargs: Mapping[str, object] = _EMPTY + ) -> ProgramBinding: + """Bind a new program, or return its existing row. + + A program is private to the key that started it: the same id under another key starts that key's + own program instead of riding on, or completing, someone else's. Concurrent first requests of one + program (a harness that fans out) wait for the decision in flight instead of each asking the + decision maker, so a program is bound exactly once. + """ + key: Final = ProgramKey(api_key_hash, program_id) + in_flight = self._binding_in_flight.get(key) # rebind-ok: re-read after each wait + while in_flight is not None: + await in_flight.wait() + in_flight = self._binding_in_flight.get(key) + existing: Final = self._bindings.get(key) + if existing is not None: + return existing + event: Final = asyncio.Event() + self._binding_in_flight[key] = event + try: + return await self._bind(key, prompt, request_kwargs) + finally: + self._binding_in_flight.pop(key, None) + event.set() + + async def _bind(self, key: ProgramKey, prompt: str, request_kwargs: Mapping[str, object]) -> ProgramBinding: + now: Final = self._clock() + if len(self._bindings) >= min(PROGRAM_SWEEP_THRESHOLD, self.config.max_programs): + self._evict_expired(now) + context: Final = self.context_for(key.program_id, prompt, request_kwargs) + model: Final = await self.decision_maker.select(context) + binding: Final = ProgramBinding( + program_id=key.program_id, + model=model, + context=replace(context, caller=_EMPTY), + api_key_hash=key.owner, + bound_at=now, + message_logging_off=context.message_logging_off, + ) + self._bindings[key] = binding + self.programs_bound += 1 + return binding + + async def async_pre_routing_hook( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: pre-routing contract + messages: list[dict[str, object]] | None = None, # mutable-ok: pre-routing contract + input: str | list[object] | None = None, # mutable-ok: pre-routing contract + specific_deployment: bool | None = False, + ) -> PreRoutingHookResponse | None: + """Route the request to its program's model, binding the program on its first request.""" + program_id: Final = resolve_program_id(request_kwargs, self.config.program_id_key) + prompt: Final = first_user_text(program_messages(messages, input, request_kwargs)) + chosen_model, request_type = await self._decide(program_id, prompt, request_kwargs) + _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) + _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) + return PreRoutingHookResponse( + model=chosen_model, + messages=messages, + routing_decision=StandardLoggingRoutingDecision( + router_model_name=self.router_name, + router_type="oracle", + routed_model=chosen_model, + cause=self._cause, + request_type=request_type.value, + ), + ) + + async def _decide( + self, program_id: str | None, prompt: str, request_kwargs: Mapping[str, object] + ) -> tuple[str, RequestType]: + if program_id is None: + self.stateless_requests += 1 + context: Final = self.context_for("", prompt, request_kwargs) + return await self.decision_maker.select(context), context.request_type + binding: Final = await self.bind(program_id, prompt, _api_key_hash(request_kwargs), request_kwargs) + return binding.model, binding.context.request_type + + def mark_message_logging_off(self, program_id: str, owner: str | None = None) -> None: + """A request of the program opted out of message logging, whether or not it produced an answer: the + whole program is private from here on, and its verification runs under that opt-out.""" + key: Final = ProgramKey(owner, program_id) + binding: Final = self._bindings.get(key) + if binding is not None and not binding.message_logging_off: + self._bindings[key] = replace(binding, message_logging_off=True) + + def observe_request( + self, program_id: str, cost: float, response_text: str, messages: object = None, owner: str | None = None + ) -> None: + """Account one finished LLM request of the program: LiteLLM's spend, the latest answer and transcript.""" + key: Final = ProgramKey(owner, program_id) + binding: Final = self._bindings.get(key) + if binding is None: + return + transcript: Final = _transcript(messages, binding.last_messages) + self._bindings[key] = replace( + binding, + requests=binding.requests + 1, + cost=binding.cost + max(cost, 0.0), + last_response_text=response_text or binding.last_response_text, + last_messages=transcript, + ) + deferred: Final = binding.deferred + if deferred is not None: # the harness reported before this response was observed + self.complete(program_id, score=deferred.score, cost=deferred.cost, payload=deferred.payload, owner=owner) + + def complete( + self, + program_id: str, + score: float | None = None, + cost: float | None = None, + payload: Mapping[str, object] = _EMPTY, + owner: str | None = None, + ) -> asyncio.Task[float] | None: + """Release the program now and verify it in the background; the task resolves to the verified score. + + A program none of whose requests has been observed yet (feedback that raced the last response's + logging, or a program whose only request failed) keeps its slot and is completed, with these + arguments, by its next observed response: there is no answer to verify yet, so nothing is learned. + + Raises ``VerificationBacklogFull`` when ``max_pending_verifications`` verifications are already in + flight: the program stays bound, so the caller can complete it again once the backlog drains. + """ + key: Final = ProgramKey(owner, program_id) + binding: Final = self._bindings.get(key) + if binding is None: + return None + if binding.requests == 0: + self._bindings[key] = replace(binding, deferred=DeferredCompletion(score=score, cost=cost, payload=payload)) + return None + if len(self._tasks) >= self.config.max_pending_verifications: + self.verifications_rejected += 1 + raise VerificationBacklogFull( + f"{len(self._tasks)} verifications are in flight (max_pending_verifications=" + f"{self.config.max_pending_verifications}); program {program_id!r} stays bound, complete it again later" + ) + del self._bindings[key] + outcome: Final = ProgramOutcome( + program_id=program_id, + model=binding.model, + prompt=binding.context.prompt, + response_text=binding.last_response_text, + cost=cost if cost is not None else binding.cost, + messages=binding.last_messages, + score=score, + payload=payload, + message_logging_off=binding.message_logging_off, + ) + self.programs_completed += 1 + task: Final = asyncio.create_task(self._verify_and_learn(binding, outcome)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + return task + + async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float: + try: + with inherit_message_logging_privacy(outcome.message_logging_off): + score: Final = clamp_score( + await asyncio.wait_for( + self.verifier.verify(outcome), timeout=self.config.verification_timeout_seconds + ) + ) + except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop + self.verifications_failed += 1 + self.note_failure("verification", error) + return float("nan") + async with self._update_lock: + self.decision_maker.update(binding.context, binding.model, score) + self._history.append( + FeedbackRecord( + program_id=binding.program_id, + model=binding.model, + request_type=binding.context.request_type.value, + score=score, + cost=outcome.cost, + verified_at=self._clock(), + ) + ) + return score + + async def drain(self) -> None: + """Wait for every in-flight verification.""" + while self._tasks: + await asyncio.gather(*tuple(self._tasks), return_exceptions=True) + + @property + def pending_verifications(self) -> int: + return len(self._tasks) + + def _accuracy_by_model(self) -> Mapping[str, float]: + scores: Final[dict[str, list[float]]] = {} # mutable-ok: grouping accumulator for the snapshot + for record in self._history: + scores.setdefault(record.model, []).append(record.score) + return {model: round(sum(values) / len(values), 4) for model, values in scores.items()} + + async def get_state_snapshot(self) -> Mapping[str, object]: + """In-memory view for the introspection endpoint.""" + active: Final[dict[str, int]] = {} # mutable-ok: counting accumulator for the snapshot + for binding in self._bindings.values(): + active[binding.model] = active.get(binding.model, 0) + 1 + return { + "router_name": self.router_name, + "available_models": list(self.models), + "decision_maker": self.decision_maker.snapshot(), + "verifier": type(self.verifier).__name__, + "active_programs": len(self._bindings), + "active_per_model": active, + "programs_bound": self.programs_bound, + "programs_completed": self.programs_completed, + "programs_evicted": self.programs_evicted, + "stateless_requests": self.stateless_requests, + "pending_verifications": self.pending_verifications, + "verifications_failed": self.verifications_failed, + "verifications_rejected": self.verifications_rejected, + "last_failure": self.last_failure, + "recent_accuracy": self._accuracy_by_model(), + "recent_feedback": [ + { + "program_id": record.program_id, + "model": record.model, + "request_type": record.request_type, + "score": record.score, + "cost": record.cost, + } + for record in tuple(self._history)[-20:] + ], + } + + +def request_type_of(prompt: str) -> RequestType: + return classify_prompt(prompt) diff --git a/litellm/router_strategy/oracle_router/verifier.py b/litellm/router_strategy/oracle_router/verifier.py new file mode 100644 index 00000000000..7a52ae6b2c0 --- /dev/null +++ b/litellm/router_strategy/oracle_router/verifier.py @@ -0,0 +1,235 @@ +"""Verifiers score a finished program in [0, 1], off the critical path. + +The contract is one coroutine. ORACLE ships three: any configured LiteLLM guardrail applied to the +program's final turn with its transcript as context (``llm_as_a_judge``, ``custom_code`` or a hosted +check), the score the client reported, and an import path for an operator's own class. +""" + +import math +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final, Protocol, runtime_checkable + +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, REDACTED_BY_LITELLM +from litellm.exceptions import GuardrailRaisedException +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import OracleVerifierConfig +from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse + +from .decision import load_object + +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_STATUS_RECORDS: Final = TypeAdapter(Sequence[Mapping[str, object]]) +_STATUS_RECORD_KEY: Final[str] = "standard_logging_guardrail_information" +_PROXY_OWNED_METADATA_KEYS: Final[frozenset[str]] = frozenset( + {_STATUS_RECORD_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "headers", "routing_decision"} +) +_BLOCKED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_intervened", "guardrail_flagged"}) +_UNVERIFIED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_failed_to_respond", "not_run"}) + + +@dataclass(frozen=True, slots=True) +class ProgramOutcome: + """Everything a verifier may look at once a program has finished.""" + + program_id: str + model: str + prompt: str + response_text: str + cost: float + messages: Sequence[AllMessageValues] = () + score: float | None = None + payload: Mapping[str, object] = field(default_factory=lambda: _EMPTY) + message_logging_off: bool = False + """Whether any request of the program opted out of message logging; the router verifies under that opt-out.""" + + +@runtime_checkable +class Verifier(Protocol): + async def verify(self, outcome: ProgramOutcome) -> float: ... + + +def clamp_score(value: float) -> float: + """Clamp a score to [0, 1]. A NaN/inf score raises: it would corrupt a bandit cell for good.""" + score: Final = float(value) + if not math.isfinite(score): + raise ValueError(f"score must be finite, got {score!r}") + return min(max(score, 0.0), 1.0) + + +def response_text(response: object) -> str: + """The assistant text of a chat completion, or an empty string for anything else.""" + if not isinstance(response, ModelResponse) or not response.choices: + return "" + content: Final = response.choices[0].message.content + return content if isinstance(content, str) else "" + + +class ReportedVerifier: + """Trust the score the client reported with the program's last request or through the feedback endpoint.""" + + def __init__(self, default: float = 0.0) -> None: + self._default: Final[float] = default + + async def verify(self, outcome: ProgramOutcome) -> float: + return clamp_score(outcome.score) if outcome.score is not None else self._default + + +def find_guardrail(name: str) -> CustomGuardrail | None: + """The initialized guardrail registered under ``name``, if any.""" + callbacks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(callback_type=CustomGuardrail) + guardrails: Final = tuple(callback for callback in callbacks if isinstance(callback, CustomGuardrail)) + return next((guardrail for guardrail in guardrails if guardrail.guardrail_name == name), None) + + +def _redacted(outcome: ProgramOutcome) -> bool: + """Whether LiteLLM replaced the program's final answer or transcript with its redaction placeholder.""" + return outcome.response_text == REDACTED_BY_LITELLM or any( + _STR_MAPPING.validate_python(message).get("content") == REDACTED_BY_LITELLM for message in outcome.messages + ) + + +def _verifier_metadata(outcome: ProgramOutcome) -> dict[str, object]: # mutable-ok: the guardrail writes into it + """The feedback payload as the guardrail's request metadata. + + The verdict is read from the status record the guardrail writes here, so the caller's payload must not + pre-seed that key (a non-record entry would make the whole list unreadable and hide a rejection), it + cannot rename the program, and it cannot pose as the proxy: the ``user_api_key_*`` identity a guardrail + may forward to its vendor, the sub-call origin and the request headers are the proxy's to stamp. + """ + return { + **{key: value for key, value in outcome.payload.items() if not _proxy_owned(key)}, + "program_id": outcome.program_id, + } + + +def _proxy_owned(key: str) -> bool: + return key.startswith("user_api_key") or key in _PROXY_OWNED_METADATA_KEYS + + +def _status_records(bucket: object) -> Sequence[Mapping[str, object]]: + """The ``standard_logging_guardrail_information`` records of one metadata bucket, or none.""" + try: + return _STATUS_RECORDS.validate_python(_STR_MAPPING.validate_python(bucket).get(_STATUS_RECORD_KEY) or ()) + except ValidationError: + return () + + +def recorded_status(request_data: Mapping[str, object], guardrail_name: str) -> str | None: + """The last status ``guardrail_name`` recorded for itself in ``request_data`` while it ran, if any. + + Every first-party guardrail writes a ``standard_logging_guardrail_information`` record into the + request's metadata bucket (``success``, ``guardrail_intervened``, ``guardrail_failed_to_respond``, + ...), which is the only way to tell a judge that failed the answer from one that failed to answer. + """ + records: Final = ( + *_status_records(request_data.get("metadata")), + *_status_records(request_data.get("litellm_metadata")), + ) + statuses: Final = tuple( + str(record["guardrail_status"]) + for record in records + if record.get("guardrail_name") in (guardrail_name, None) and record.get("guardrail_status") + ) + return statuses[-1] if statuses else None + + +async def _apply( + guardrail: CustomGuardrail, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], # mutable-ok: the guardrail writes its status into it +) -> Exception | None: + """Run the guardrail; the exception it raised to signal its verdict, or ``None`` when it returned.""" + try: + await guardrail.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # upstream types request_data as a bare dict + inputs=inputs, request_data=request_data, input_type="response" + ) + except Exception as error: # noqa: BLE001 # a guardrail may signal its verdict by raising; the caller classifies it + return error + return None + + +def _is_http_block(error: Exception) -> bool: + """A guardrail without a status record that blocks by raising FastAPI's ``HTTPException``.""" + try: + from fastapi import HTTPException + except ImportError: # the SDK without the proxy extra: nothing can raise it + return False + return isinstance(error, HTTPException) + + +class GuardrailVerifier: + """Apply a configured LiteLLM guardrail to the program's final turn: it passes or it is blocked. + + The guardrail sees the final assistant text as ``texts``, the program's transcript as + ``structured_messages`` and the feedback payload as ``request_data["metadata"]``, so an + ``llm_as_a_judge`` guardrail judges the whole conversation and a ``custom_code`` guardrail can read + whatever the harness posted with the feedback. + + The verdict is read from the status the guardrail records for itself (so a judge configured with + ``on_failure: log`` still scores 0 when the answer fails), and falls back to how it raised: a + ``GuardrailRaisedException`` with ``blocked_content`` or an ``HTTPException`` is a block. A + guardrail that could not evaluate the program (``guardrail_failed_to_respond``, an unreachable + backend) raises, so the program counts as a failed verification and teaches the decision maker + nothing instead of passing it. + """ + + def __init__(self, guardrail_name: str, lookup: Callable[[str], CustomGuardrail | None] = find_guardrail) -> None: + self._name: Final[str] = guardrail_name + self._lookup: Final = lookup + + async def verify(self, outcome: ProgramOutcome) -> float: + guardrail: Final = self._lookup(self._name) + if guardrail is None: + raise LookupError(f"guardrail {self._name!r} is not initialized on this proxy") + if not outcome.response_text.strip(): + # A judge skips an empty answer without recording a verdict; that must not read as a pass. + raise RuntimeError(f"guardrail {self._name!r}: the program has no response text to verify") + if _redacted(outcome): + # The request opted out of message logging, so the router only ever saw LiteLLM's placeholder. + raise RuntimeError(f"guardrail {self._name!r}: the program's final turn was redacted before verification") + transcript: Final = list(outcome.messages) or [{"role": "user", "content": outcome.prompt}] + inputs: Final = GenericGuardrailAPIInputs( + texts=[outcome.response_text], model=outcome.model, structured_messages=transcript + ) + request_data: Final[dict[str, object]] = { # mutable-ok: the guardrail records its status into it + "model": outcome.model, + "metadata": _verifier_metadata(outcome), + } + verdict_error: Final = await _apply(guardrail, inputs, request_data) + status: Final = recorded_status(request_data, self._name) + if status in _UNVERIFIED_STATUSES: + raise RuntimeError(f"guardrail {self._name!r} did not evaluate program {outcome.program_id!r}: {status}") + if status in _BLOCKED_STATUSES: + return 0.0 + if status == "success" or verdict_error is None: + return 1.0 + if isinstance(verdict_error, GuardrailRaisedException): + if verdict_error.blocked_content: + return 0.0 + raise verdict_error # the guardrail could not reach a verdict + if _is_http_block(verdict_error): + return 0.0 + raise verdict_error + + +def build_verifier(config: OracleVerifierConfig) -> Verifier: + match config.type: + case "reported": + return ReportedVerifier(default=config.default_score) + case "guardrail": + return GuardrailVerifier(config.guardrail or "") + case "custom": + factory: Final = load_object(config.path or "") + if not callable(factory): + raise TypeError(f"oracle_router verifier.path {config.path!r} is not callable") + instance: Final = factory() + if not isinstance(instance, Verifier): + raise TypeError(f"{config.path!r} must build an object with an async verify(outcome) method") + return instance diff --git a/litellm/types/management_endpoints/oracle_router_endpoints.py b/litellm/types/management_endpoints/oracle_router_endpoints.py new file mode 100644 index 00000000000..eac548bc341 --- /dev/null +++ b/litellm/types/management_endpoints/oracle_router_endpoints.py @@ -0,0 +1,31 @@ +"""Types for the ORACLE router management endpoints.""" + +from collections.abc import Mapping, Sequence + +from pydantic import BaseModel, Field + + +class OracleRouterFeedbackRequest(BaseModel, frozen=True): + """Report that a program finished. ``score`` is optional when the router's verifier grades it.""" + + program_id: str = Field(min_length=1) + score: float | None = Field(default=None, ge=0.0, le=1.0) + cost: float | None = Field( + default=None, ge=0.0, description="Overrides the spend LiteLLM accumulated for the program" + ) + payload: Mapping[str, object] = Field(default_factory=dict, description="Passed to the verifier unchanged") + + +class OracleRouterFeedbackResponse(BaseModel, frozen=True): + program_id: str + router_name: str + model: str + pending_verifications: int + deferred: bool = Field( + default=False, + description="No response of the program has been observed yet; its next one completes it with this feedback", + ) + + +class OracleRouterStateResponse(BaseModel, frozen=True): + routers: Sequence[Mapping[str, object]] diff --git a/litellm/types/router.py b/litellm/types/router.py index 4e3e4d92b3c..2ca2b411a21 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -675,6 +675,8 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): # adaptive-router params adaptive_router_default_model: str | None = None adaptive_router_config: dict | None = None + # oracle-router params + oracle_router_config: Mapping[str, object] | None = None # quality-router params quality_router_config: dict | None = None quality_router_default_model: str | None = None @@ -1531,6 +1533,82 @@ class AdaptiveRouterPreferences(LiteLLMBaseModel): strengths: list[RequestType] = Field(default_factory=list) +OracleDecisionMakerType: TypeAlias = Literal["thompson", "fixed", "pre_routing", "custom"] +OracleVerifierType: TypeAlias = Literal["reported", "guardrail", "custom"] + + +class OracleDecisionMakerConfig(BaseModel, frozen=True): + """Which policy binds a program to a model, and its knobs. + + ``thompson`` (default) is the adaptive router's bandit trained on verified program outcomes; ``weights`` + are that bandit's quality/cost weights. ``fixed`` always picks ``model``. ``pre_routing`` lets the strategy + router named by ``router`` (a complexity, quality, adaptive or semantic router in the same model_list) + propose the model once per program. ``custom`` imports ``path`` (``package.module:Factory``) and calls it + with the model tuple; it is honoured only for deployments in the proxy config file, never for models + written through the API. + """ + + type: OracleDecisionMakerType = "thompson" + weights: AdaptiveRouterWeights = Field(default_factory=AdaptiveRouterWeights) + seed: int = 0 + model: str | None = None + router: str | None = None + path: str | None = None + + @model_validator(mode="after") + def _required_fields(self) -> "OracleDecisionMakerConfig": + required: Final = {"fixed": self.model, "pre_routing": self.router, "custom": self.path} + if self.type in required and not required[self.type]: + names: Final = {"fixed": "model", "pre_routing": "router", "custom": "path"} + raise ValueError(f"decision_maker.type={self.type!r} requires decision_maker.{names[self.type]}") + return self + + +class OracleVerifierConfig(BaseModel, frozen=True): + """How a finished program is scored. + + ``guardrail`` applies the configured LiteLLM guardrail ``guardrail`` (for example an ``llm_as_a_judge`` + guardrail) to the program's final turn with the transcript as context. ``reported`` trusts + ``metadata.program_score`` or the feedback endpoint. ``custom`` imports ``path`` and calls it with no + arguments; it is honoured only for deployments in the proxy config file, never for models written + through the API. + """ + + type: OracleVerifierType = "reported" + default_score: float = Field(default=0.0, ge=0.0, le=1.0) + guardrail: str | None = None + path: str | None = None + + @model_validator(mode="after") + def _required_fields(self) -> "OracleVerifierConfig": + required: Final = {"guardrail": self.guardrail, "custom": self.path} + if self.type in required and not required[self.type]: + names: Final = {"guardrail": "guardrail", "custom": "path"} + raise ValueError(f"verifier.type={self.type!r} requires verifier.{names[self.type]}") + return self + + +class OracleRouterConfig(BaseModel, frozen=True): + """``litellm_params.oracle_router_config`` of an ``auto_router/oracle_router`` deployment.""" + + available_models: tuple[str, ...] = Field(min_length=1) + decision_maker: OracleDecisionMakerConfig = Field(default_factory=OracleDecisionMakerConfig) + verifier: OracleVerifierConfig = Field(default_factory=OracleVerifierConfig) + program_id_key: str = "program_id" + program_ttl_seconds: int = Field(default=24 * 3600, ge=1) + max_programs: int = Field(default=10_000, ge=1) + max_pending_verifications: int = Field(default=64, ge=1) + verification_timeout_seconds: float = Field(default=120.0, gt=0) + + @model_validator(mode="after") + def _fixed_model_is_available(self) -> "OracleRouterConfig": + if len(self.available_models) != len(frozenset(self.available_models)): + raise ValueError("available_models must be distinct") + if self.decision_maker.type == "fixed" and self.decision_maker.model not in self.available_models: + raise ValueError("decision_maker.model must be one of available_models") + return self + + def reject_server_owned_wif_params(body: Mapping[str, object]) -> None: """Raise ``ValueError`` if a mapping that did not come from deployment config carries a server-owned workload identity federation or OAuth token-exchange field, or an ``api_key`` diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4c47489e08f..722a14b797f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3373,7 +3373,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" router_model_name: str - router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]] + router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic", "oracle"]] router_config_id: ReadOnly[str] router_config_updated_at: ReadOnly[str] router_config_fingerprint: ReadOnly[str] diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 44c43713438..490932b1af8 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -1085,7 +1085,7 @@ class TestClearCache: Test that clear_cache successfully clears router model caches and reloads models. """ mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = ["openai/gpt-4o", "openai/gpt-4o-mini"] @@ -1125,7 +1125,7 @@ class TestClearCache: # entries are auto_router/* deployments (so their router-map entries should be # cleared for reload); the config-defined router is preserved. mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1209,7 +1209,7 @@ class TestClearCache: ) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1270,7 +1270,7 @@ class TestClearCachePreservesConfigRouters: ) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1315,7 +1315,7 @@ class TestClearCachePreservesConfigRouters: from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1354,7 +1354,7 @@ class TestClearCachePreservesConfigRouters: from litellm.proxy.management_endpoints.model_management_endpoints import clear_cache mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.registry.return_value.all.return_value = [ { @@ -1367,11 +1367,17 @@ class TestClearCachePreservesConfigRouters: "model_info": {"id": "db-a", "db_model": True}, "litellm_params": {"model": "auto_router/adaptive_router/a1"}, }, + { + "model_name": "o1", + "model_info": {"id": "db-o", "db_model": True}, + "litellm_params": {"model": "auto_router/oracle_router/o1"}, + }, ] plugins["auto"] = {} plugins["complexity"] = {} plugins["quality"] = {"q1": MagicMock()} plugins["adaptive"] = {"a1": MagicMock()} + plugins["oracle"] = {"o1": MagicMock()} mock_config = MagicMock() mock_config.add_deployment_locked = AsyncMock( @@ -1389,6 +1395,7 @@ class TestClearCachePreservesConfigRouters: assert "q1" not in plugins["quality"] assert "a1" not in plugins["adaptive"] + assert "o1" not in plugins["oracle"] class TestDeleteModelClearsRouterRegistry: @@ -1503,7 +1510,7 @@ class TestDeleteModelClearsRouterRegistry: mock_prisma.db.litellm_proxymodeltable.delete = AsyncMock(return_value=db_row) mock_router = MagicMock() - plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}} + plugins: dict[str, dict] = {"auto": {}, "complexity": {}, "adaptive": {}, "quality": {}, "oracle": {}} mock_router.registry.return_value.plugins.side_effect = plugins.__getitem__ mock_router.apply.return_value = ApplyResult( version=1, diff --git a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py new file mode 100644 index 00000000000..da9236b2c6c --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py @@ -0,0 +1,190 @@ +"""POST /oracle_router/feedback and GET /oracle_router/state.""" + +import asyncio +from unittest.mock import patch + +import pytest +from fastapi import HTTPException +from pydantic import ValidationError + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.management_endpoints.oracle_router_endpoints import ( + get_oracle_router_state, + submit_oracle_router_feedback, +) +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter +from litellm.router_strategy.oracle_router.verifier import ReportedVerifier +from litellm.types.management_endpoints.oracle_router_endpoints import OracleRouterFeedbackRequest +from litellm.types.router import OracleRouterConfig, TaggedPreRoutingStrategy + +ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="admin-hash") +OWNER = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="owner-hash") +STRANGER = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="other-hash") + + +class _Maker: + def __init__(self) -> None: + self.updates = [] + + @property + def models(self): + return ("smart", "fast") + + async def select(self, context): + return "smart" + + def update(self, context, model, score): + self.updates.append((model, score)) + + def snapshot(self): + return {"type": "test"} + + +class _Registry: + def __init__(self, oracle: OracleRouter) -> None: + self._oracle_routers = {oracle.router_name: [TaggedPreRoutingStrategy(tags=(), strategy=oracle)]} + + def plugins(self, kind: str): + assert kind == "oracle" + return self._oracle_routers + + +class _LLMRouter: + def __init__(self, oracle: OracleRouter) -> None: + self._registry = _Registry(oracle) + + def registry(self) -> _Registry: + return self._registry + + +@pytest.fixture +def oracle(): + maker = _Maker() + router = OracleRouter("oracle", OracleRouterConfig(available_models=["smart", "fast"]), maker, ReportedVerifier()) + router.maker = maker + return router + + +async def _bind(oracle: OracleRouter, program_id: str, api_key_hash: str | None = "owner-hash"): + metadata = {"program_id": program_id, **({"user_api_key_hash": api_key_hash} if api_key_hash else {})} + await oracle.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": metadata}, messages=[{"role": "user", "content": "t"}] + ) + + +@pytest.mark.asyncio +async def test_feedback_completes_the_program_and_the_router_learns(oracle): + await _bind(oracle, "task-1") + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER + ) + assert response.model_dump() == { + "program_id": "task-1", + "router_name": "oracle", + "model": "smart", + "pending_verifications": 1, + "deferred": False, + } + assert oracle.binding("task-1", "owner-hash") is None + await oracle.drain() + assert oracle.maker.updates == [("smart", 1.0)] + + +@pytest.mark.asyncio +async def test_feedback_is_rejected_with_503_while_the_verification_backlog_is_full(): + class _Stuck: + async def verify(self, outcome): + await asyncio.sleep(5.0) + return 1.0 + + config = OracleRouterConfig(available_models=["smart", "fast"], max_pending_verifications=1) + oracle = OracleRouter("oracle", config, _Maker(), _Stuck()) + for program_id in ("task-1", "task-2"): + await _bind(oracle, program_id) + oracle.observe_request(program_id, cost=0.2, response_text="done", owner="owner-hash") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER) + with pytest.raises(HTTPException) as rejected: + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-2", score=1.0), OWNER) + assert rejected.value.status_code == 503 and "max_pending_verifications=1" in rejected.value.detail["error"] + assert oracle.binding("task-2", "owner-hash") is not None + for task in tuple(oracle._tasks): + task.cancel() + + +@pytest.mark.asyncio +async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it(oracle): + await _bind(oracle, "task-1") + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + with pytest.raises(HTTPException) as unseen: + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), STRANGER) + assert unseen.value.status_code == 404 and oracle.binding("task-1", "owner-hash") is not None + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=0.0), ADMIN + ) + assert response.program_id == "task-1" and oracle.binding("task-1", "owner-hash") is None + await oracle.drain() + + +@pytest.mark.asyncio +async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across_keys(oracle): + await _bind(oracle, "task-1", "owner-hash") + await _bind(oracle, "task-1", "other-hash") + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="other-hash") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=0.5), STRANGER + ) + assert response.program_id == "task-1" + assert oracle.binding("task-1", "other-hash") is None and oracle.binding("task-1", "owner-hash") is not None + await oracle.drain() + + +@pytest.mark.asyncio +async def test_feedback_before_any_response_is_deferred_until_one_is_observed(oracle): + await _bind(oracle, "task-1") # bound, but its first response has not been observed yet + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER + ) + assert response.deferred is True and response.pending_verifications == 0 + assert oracle.binding("task-1", "owner-hash") is not None and oracle.maker.updates == [] + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash") + assert oracle.binding("task-1", "owner-hash") is None + await oracle.drain() + assert oracle.maker.updates == [("smart", 1.0)] + + +@pytest.mark.asyncio +async def test_feedback_for_an_unknown_program_or_without_routers_is_404(oracle): + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + with pytest.raises(HTTPException) as unknown: + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="nope"), OWNER) + assert unknown.value.status_code == 404 + with patch("litellm.proxy.proxy_server.llm_router", None): + with pytest.raises(HTTPException) as none: + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="nope"), OWNER) + assert none.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_state_is_admin_only_and_lists_every_router(oracle): + await _bind(oracle, "task-1") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + with pytest.raises(HTTPException) as denied: + await get_oracle_router_state(OWNER) + assert denied.value.status_code == 403 + state = await get_oracle_router_state(ADMIN) + assert len(state.routers) == 1 + assert state.routers[0]["router_name"] == "oracle" and state.routers[0]["active_programs"] == 1 + assert state.routers[0]["decision_maker"] == {"type": "test"} + + +def test_feedback_request_validates_score_range(): + with pytest.raises(ValidationError, match="score"): + OracleRouterFeedbackRequest(program_id="p", score=1.5) + with pytest.raises(ValidationError, match="program_id"): + OracleRouterFeedbackRequest(program_id="") diff --git a/tests/unit/router_strategy/oracle_router/__init__.py b/tests/unit/router_strategy/oracle_router/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/router_strategy/oracle_router/test_decision.py b/tests/unit/router_strategy/oracle_router/test_decision.py new file mode 100644 index 00000000000..77f4bdfc537 --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_decision.py @@ -0,0 +1,210 @@ +"""Decision makers: each learns or proposes the way its contract says.""" + +import pytest + +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, +) +from litellm.litellm_core_utils.internal_call_metadata import effective_turn_off_message_logging +from litellm.router_strategy.oracle_router.decision import ( + DecisionMaker, + FixedDecisionMaker, + PreRoutingDecisionMaker, + ProgramContext, + ThompsonDecisionMaker, + build_decision_maker, + load_object, +) +from litellm.types.router import AdaptiveRouterWeights, OracleDecisionMakerConfig, PreRoutingHookResponse, RequestType + +MODELS = ("smart", "fast") +COSTS = {"smart": 1e-5, "fast": 1e-6} + + +def _ctx(prompt: str = "fix the failing test", request_type: RequestType = RequestType.GENERAL) -> ProgramContext: + return ProgramContext(program_id="p", prompt=prompt, request_type=request_type) + + +@pytest.mark.asyncio +async def test_thompson_learns_a_different_winner_per_request_type(): + maker = ThompsonDecisionMaker(MODELS, COSTS, weights=AdaptiveRouterWeights(quality=1.0, cost=0.0), seed=0) + for _ in range(80): + for request_type, good in ((RequestType.CODE_GENERATION, "smart"), (RequestType.WRITING, "fast")): + ctx = _ctx(request_type=request_type) + chosen = await maker.select(ctx) + maker.update(ctx, chosen, 1.0 if chosen == good else 0.0) + code, writing = _ctx(request_type=RequestType.CODE_GENERATION), _ctx(request_type=RequestType.WRITING) + assert maker.estimate(code, "smart") > maker.estimate(code, "fast") + assert maker.estimate(writing, "fast") > maker.estimate(writing, "smart") + picks = [await maker.select(code) for _ in range(30)] + assert picks.count("smart") > 24 + assert maker.snapshot()["cells"]["code_generation/smart"]["samples"] > 0 + + +@pytest.mark.asyncio +async def test_thompson_cost_weight_prefers_the_cheap_model_when_quality_ties(): + maker = ThompsonDecisionMaker(MODELS, COSTS, weights=AdaptiveRouterWeights(quality=0.3, cost=0.7), seed=1) + ctx = _ctx() + for _ in range(40): + maker.update(ctx, "smart", 0.8) + maker.update(ctx, "fast", 0.8) + picks = [await maker.select(ctx) for _ in range(30)] + assert picks.count("fast") > 24 + + +@pytest.mark.asyncio +async def test_thompson_priors_come_from_adaptive_router_preferences(): + from litellm.types.router import AdaptiveRouterPreferences + + prefs = { + "smart": AdaptiveRouterPreferences(quality_tier=3, strengths=[]), + "fast": AdaptiveRouterPreferences(quality_tier=1, strengths=[]), + } + maker = ThompsonDecisionMaker( + MODELS, COSTS, weights=AdaptiveRouterWeights(quality=1.0, cost=0.0), model_prefs=prefs, seed=0 + ) + ctx = _ctx() + assert maker.estimate(ctx, "smart") > maker.estimate(ctx, "fast") + picks = [await maker.select(ctx) for _ in range(30)] + assert picks.count("smart") > 24 + + +def test_thompson_rejects_unknown_model_and_empty_pool(): + maker = ThompsonDecisionMaker(MODELS, COSTS) + with pytest.raises(KeyError): + maker.update(_ctx(), "other", 1.0) + with pytest.raises(ValueError, match="at least one model"): + ThompsonDecisionMaker((), COSTS) + + +@pytest.mark.asyncio +async def test_fixed_always_returns_its_model_and_counts_updates(): + maker = FixedDecisionMaker(MODELS, model="fast") + assert await maker.select(_ctx()) == "fast" + maker.update(_ctx(), "fast", 1.0) + assert maker.snapshot() == {"type": "fixed", "model": "fast", "updates": 1} + with pytest.raises(KeyError): + FixedDecisionMaker(MODELS, model="nope") + + +class _FakeStrategy: + def __init__(self, answer: str | None) -> None: + self.answer = answer + self.calls: list[dict] = [] + + async def async_pre_routing_hook(self, model, request_kwargs, messages=None, input=None, specific_deployment=False): + self.calls.append({"model": model, "messages": messages}) + self.request_kwargs = request_kwargs + self.privacy = ( + effective_turn_off_message_logging(request_kwargs), + initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging"), + ) + return None if self.answer is None else PreRoutingHookResponse(model=self.answer, messages=messages) + + +@pytest.mark.asyncio +async def test_pre_routing_uses_the_wrapped_strategy_proposal_once_per_program(): + strategy = _FakeStrategy("fast") + maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy) + assert await maker.select(_ctx("write a haiku")) == "fast" + assert strategy.calls == [{"model": "tiered", "messages": [{"role": "user", "content": "write a haiku"}]}] + assert maker.snapshot()["proposals"] == {"fast": 1} + + +@pytest.mark.asyncio +async def test_pre_routing_forwards_the_program_s_message_logging_opt_out_to_the_wrapped_strategy(): + strategy = _FakeStrategy("fast") + maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy) + private = ProgramContext( + program_id="p", prompt="secret", request_type=RequestType.GENERAL, message_logging_off=True + ) + assert await maker.select(private) == "fast" + assert strategy.privacy == (True, True) + assert await maker.select(_ctx("public")) == "fast" + assert strategy.privacy == (None, None) + + +@pytest.mark.asyncio +async def test_pre_routing_forwards_the_caller_s_identity_so_a_classifier_call_is_billed_to_the_caller(): + strategy = _FakeStrategy("fast") + maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy) + caller = {"user_api_key_hash": "hash-1", "user_api_key_team_id": "team-1", "user_api_key_user_id": "user-1"} + private = ProgramContext( + program_id="p", prompt="task", request_type=RequestType.GENERAL, message_logging_off=True, caller=caller + ) + assert await maker.select(private) == "fast" + assert strategy.request_kwargs == {"metadata": {**caller, "turn_off_message_logging": True}} + first_call = strategy.request_kwargs + assert await maker.select(private) == "fast" + assert strategy.request_kwargs == first_call and strategy.request_kwargs is not first_call + assert await maker.select(_ctx()) == "fast" + assert strategy.request_kwargs == {"metadata": {}} + + +@pytest.mark.asyncio +async def test_pre_routing_falls_back_to_the_first_model_when_the_proposal_is_unusable(): + assert await PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy(None)).select(_ctx()) == "smart" + maker = PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy("not-configured")) + assert await maker.select(_ctx()) == "smart" + assert maker.snapshot()["fallbacks"] == 1 + + +class CustomMaker: + def __init__(self, models: tuple[str, ...]) -> None: + self._models = models + + @property + def models(self) -> tuple[str, ...]: + return self._models + + async def select(self, context: ProgramContext) -> str: + return self._models[-1] + + def update(self, context: ProgramContext, model: str, score: float) -> None: + pass + + def snapshot(self): + return {"type": "custom"} + + +class NotAMaker: + def __init__(self, models: tuple[str, ...]) -> None: + pass + + +@pytest.mark.asyncio +async def test_build_decision_maker_by_type(): + resolve = {"tiered": _FakeStrategy("smart")}.get + thompson = build_decision_maker(OracleDecisionMakerConfig(), MODELS, COSTS, resolve) + assert isinstance(thompson, ThompsonDecisionMaker) and await thompson.select(_ctx()) in MODELS + assert isinstance( + build_decision_maker(OracleDecisionMakerConfig(type="fixed", model="fast"), MODELS, COSTS, resolve), + FixedDecisionMaker, + ) + pre = build_decision_maker(OracleDecisionMakerConfig(type="pre_routing", router="tiered"), MODELS, COSTS, resolve) + assert isinstance(pre, PreRoutingDecisionMaker) and await pre.select(_ctx()) == "smart" + with pytest.raises(ValueError, match="not a configured strategy router"): + build_decision_maker(OracleDecisionMakerConfig(type="pre_routing", router="missing"), MODELS, COSTS, resolve) + custom = build_decision_maker( + OracleDecisionMakerConfig(type="custom", path=f"{__name__}:CustomMaker"), MODELS, COSTS, resolve + ) + assert isinstance(custom, DecisionMaker) and await custom.select(_ctx()) == "fast" + with pytest.raises(TypeError): + build_decision_maker( + OracleDecisionMakerConfig(type="custom", path=f"{__name__}:NotAMaker"), MODELS, COSTS, resolve + ) + + +def test_load_object_requires_module_colon_attribute(): + with pytest.raises(ValueError, match=r"package\.module:Attribute"): + load_object("no_colon_here") + assert load_object(f"{__name__}:CustomMaker") is CustomMaker + + +def test_config_requires_the_fields_each_type_reads(): + with pytest.raises(ValueError, match=r"decision_maker\.model"): + OracleDecisionMakerConfig(type="fixed") + with pytest.raises(ValueError, match=r"decision_maker\.router"): + OracleDecisionMakerConfig(type="pre_routing") + with pytest.raises(ValueError, match=r"decision_maker\.path"): + OracleDecisionMakerConfig(type="custom") diff --git a/tests/unit/router_strategy/oracle_router/test_hooks.py b/tests/unit/router_strategy/oracle_router/test_hooks.py new file mode 100644 index 00000000000..55299fc72ec --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_hooks.py @@ -0,0 +1,200 @@ +"""The post-call hook feeds spend and completion signals from LiteLLM's logging payload into the router.""" + +import asyncio + +import pytest + +from litellm.router_strategy.oracle_router.config import ( + CHOSEN_MODEL_METADATA_KEY, + PROGRAM_ID_METADATA_KEY, + RESPONSE_HEADER, +) +from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter +from litellm.router_strategy.oracle_router.verifier import ReportedVerifier +from litellm.types.router import OracleRouterConfig +from litellm.types.utils import Choices, Message, ModelResponse + + +class _Maker: + def __init__(self) -> None: + self.updates = [] + + @property + def models(self): + return ("smart", "fast") + + async def select(self, context): + return "smart" + + def update(self, context, model, score): + self.updates.append((model, score)) + + def snapshot(self): + return {} + + +ROUTED = {"router_model_name": "oracle", "router_type": "oracle"} # what the Router stamps for this router + + +def _kwargs(program_id="t1", cost=0.01, routing_decision=ROUTED, **metadata): + stamped = {PROGRAM_ID_METADATA_KEY: program_id, **metadata} + if routing_decision is not None: + stamped["routing_decision"] = routing_decision + return { + "model": "openai/gpt-4o", + "messages": [{"role": "user", "content": "task"}], + "litellm_params": {"metadata": stamped}, + "response_cost": cost, + } + + +def _response(text="answer") -> ModelResponse: + return ModelResponse(choices=[Choices(message=Message(content=text, role="assistant"))]) + + +@pytest.fixture +def bound(): + maker = _Maker() + router = OracleRouter("oracle", OracleRouterConfig(available_models=["smart", "fast"]), maker, ReportedVerifier()) + asyncio.run( + router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "t1"}}, + messages=[{"role": "user", "content": "task"}], + ) + ) + return router, maker, OracleRouterPostCallHook(router) + + +@pytest.mark.asyncio +async def test_success_events_accumulate_spend_and_the_latest_answer(bound): + router, _, hook = bound + await hook.async_log_success_event(_kwargs(cost=0.01), _response("first"), None, None) + await hook.async_log_success_event(_kwargs(cost=0.02), _response("second"), None, None) + binding = router.binding("t1") + assert binding.requests == 2 and binding.cost == pytest.approx(0.03) and binding.last_response_text == "second" + assert list(binding.last_messages) == [{"role": "user", "content": "task"}] + + +@pytest.mark.asyncio +async def test_a_turn_that_opted_out_of_message_logging_marks_the_program_private_for_good(bound): + router, _, hook = bound + await hook.async_log_success_event(_kwargs(cost=0.01), _response("first"), None, None) + assert router.binding("t1").message_logging_off is False + await hook.async_log_success_event( + _kwargs(cost=0.01, turn_off_message_logging=True), _response("second"), None, None + ) + assert router.binding("t1").message_logging_off is True + await hook.async_log_success_event(_kwargs(cost=0.01), _response("third"), None, None) + assert router.binding("t1").message_logging_off is True + + +@pytest.mark.asyncio +async def test_a_private_turn_that_failed_still_marks_the_program_private(bound): + router, _, hook = bound + await hook.async_log_failure_event(_kwargs(cost=0.5, turn_off_message_logging=True), None, None, None) + assert router.binding("t1").message_logging_off is True and router.binding("t1").requests == 0 + + +@pytest.mark.asyncio +async def test_a_turn_litellm_resolved_as_private_marks_the_program_private(bound): + router, _, hook = bound + resolved = {**_kwargs(cost=0.01), "standard_callback_dynamic_params": {"turn_off_message_logging": True}} + await hook.async_log_success_event(resolved, _response("first"), None, None) + assert router.binding("t1").message_logging_off is True + + +@pytest.mark.asyncio +async def test_program_done_completes_the_program_with_the_reported_score(bound): + router, maker, hook = bound + await hook.async_log_success_event( + _kwargs(cost=0.05, program_done=True, program_score="1"), _response(), None, None + ) + assert router.binding("t1") is None + await router.drain() + assert maker.updates == [("smart", 1.0)] + + +@pytest.mark.asyncio +async def test_a_non_finite_program_score_is_treated_as_no_score(bound): + router, maker, hook = bound + await hook.async_log_success_event( + _kwargs(cost=0.05, program_done=True, program_score="nan"), _response(), None, None + ) + assert router.binding("t1") is None + await router.drain() + assert maker.updates == [("smart", 0.0)] # the reported verifier's default, never NaN + + +@pytest.mark.asyncio +async def test_failure_event_adds_no_spend_and_program_done_on_it_waits_for_a_response(bound): + router, maker, hook = bound + await hook.async_log_failure_event(_kwargs(cost=0.5), None, None, None) + assert router.binding("t1").cost == 0.0 + await hook.async_log_failure_event(_kwargs(program_done=True), None, None, None) + assert router.binding("t1") is not None and maker.updates == [] # no answer to verify yet: deferred + await hook.async_log_success_event(_kwargs(cost=0.1), _response("late answer"), None, None) + assert router.binding("t1") is None + await router.drain() + assert maker.updates == [("smart", 0.0)] + + +@pytest.mark.asyncio +async def test_events_for_unknown_or_untagged_programs_are_ignored(bound): + router, _, hook = bound + await hook.async_log_success_event( + {"litellm_params": {"metadata": {}}, "response_cost": 1.0}, _response(), None, None + ) + await hook.async_log_success_event(_kwargs(program_id="other"), _response(), None, None) + assert router.binding("t1").requests == 0 and router.binding("other") is None + + +@pytest.mark.asyncio +async def test_events_from_requests_this_router_did_not_route_are_ignored(bound): + router, maker, hook = bound + unrouted = _kwargs(cost=0.5, program_done=True, program_score="1", routing_decision=None) + await hook.async_log_success_event(unrouted, _response(), None, None) + elsewhere = _kwargs( + cost=0.5, program_done=True, program_score="1", routing_decision={**ROUTED, "router_model_name": "x"} + ) + await hook.async_log_success_event(elsewhere, _response(), None, None) + assert router.binding("t1").requests == 0 and maker.updates == [] + + +@pytest.mark.asyncio +async def test_events_only_reach_the_program_of_the_key_that_sent_them(bound): + router, maker, hook = bound # t1 was started without a key + stranger = _kwargs(cost=0.5, program_done=True, program_score="1", user_api_key_hash="other-key") + await hook.async_log_success_event(stranger, _response(), None, None) + assert router.binding("t1").requests == 0 and maker.updates == [] + await router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "t1", "user_api_key_hash": "other-key"}}, + messages=[{"role": "user", "content": "task"}], + ) + await hook.async_log_success_event(stranger, _response(), None, None) + assert router.binding("t1", "other-key") is None and router.binding("t1").requests == 0 + await router.drain() + assert maker.updates == [("smart", 1.0)] + + +@pytest.mark.asyncio +async def test_response_header_names_the_bound_model(bound): + _, _, hook = bound + headers = await hook.async_post_call_response_headers_hook( + {"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart", "routing_decision": ROUTED}}, None, None + ) + assert headers == {RESPONSE_HEADER: "smart"} + assert ( + await hook.async_post_call_response_headers_hook({"metadata": {"routing_decision": ROUTED}}, None, None) is None + ) + forged = {"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart"}} # a request this router did not route + assert await hook.async_post_call_response_headers_hook(forged, None, None) is None + + +@pytest.mark.asyncio +async def test_response_header_reads_the_litellm_metadata_bucket_too(bound): + _, _, hook = bound + data = {"metadata": {}, "litellm_metadata": {CHOSEN_MODEL_METADATA_KEY: "fast", "routing_decision": ROUTED}} + assert await hook.async_post_call_response_headers_hook(data, None, None) == {RESPONSE_HEADER: "fast"} diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py new file mode 100644 index 00000000000..0b51c6e3f9e --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -0,0 +1,445 @@ +"""OracleRouter: sticky program bindings, program id resolution, delayed feedback, eviction.""" + +import asyncio +import math + +import pytest + +from litellm.litellm_core_utils.core_helpers import budget_reservation_from_metadata +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, +) +from litellm.router_strategy.oracle_router.config import ( + CHOSEN_MODEL_METADATA_KEY, + PROGRAM_ID_HEADER, + PROGRAM_ID_METADATA_KEY, +) +from litellm.router_strategy.oracle_router.decision import ProgramContext +from litellm.router_strategy.oracle_router.oracle_router import ( + OracleRouter, + VerificationBacklogFull, + first_user_text, + resolve_program_id, +) +from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, ReportedVerifier +from litellm.types.router import OracleRouterConfig + +MODELS = ("smart", "fast") + + +class _RecordingMaker: + """Alternates models per bind and records every update in arrival order.""" + + def __init__(self) -> None: + self.selections = 0 + self.updates: list[tuple[str, str, float]] = [] + self.contexts: list[ProgramContext] = [] + + @property + def models(self): + return MODELS + + async def select(self, context: ProgramContext) -> str: + self.selections += 1 + self.contexts.append(context) + return MODELS[(self.selections - 1) % 2] + + def update(self, context: ProgramContext, model: str, score: float) -> None: + self.updates.append((context.program_id, model, score)) + + def snapshot(self): + return {"type": "recording"} + + +class _SlowVerifier: + def __init__(self, delays: dict[str, float]) -> None: + self.delays = delays + + async def verify(self, outcome: ProgramOutcome) -> float: + await asyncio.sleep(self.delays.get(outcome.program_id, 0.0)) + return outcome.score if outcome.score is not None else 0.0 + + +def _router(verifier=None, maker=None, clock=None, **config) -> tuple[OracleRouter, _RecordingMaker]: + maker = maker or _RecordingMaker() + router = OracleRouter( + router_name="oracle", + config=OracleRouterConfig(available_models=list(MODELS), **config), + decision_maker=maker, + verifier=verifier or ReportedVerifier(), + **({"clock": clock} if clock else {}), + ) + return router, maker + + +def _messages(text: str = "fix the failing test in src/parser.py"): + return [{"role": "system", "content": "be terse"}, {"role": "user", "content": text}] + + +def test_resolve_program_id_precedence_and_header(): + assert resolve_program_id({"metadata": {"program_id": "a", "litellm_session_id": "b"}}, "program_id") == "a" + assert resolve_program_id({"metadata": {"litellm_session_id": "b"}}, "program_id") == "b" + assert resolve_program_id({"metadata": {"session_id": 7}}, "program_id") == "7" + assert resolve_program_id({"litellm_session_id": "lp"}, "program_id") == "lp" + assert resolve_program_id({"headers": {PROGRAM_ID_HEADER.upper(): "h"}}, "program_id") == "h" + assert resolve_program_id({"metadata": {"task": "t"}}, "task") == "t" + assert resolve_program_id({"metadata": {"program_id": ""}}, "program_id") is None + + +def test_first_user_text_reads_the_initial_request_including_text_parts(): + assert first_user_text(_messages("hello")) == "hello" + parts = [ + { + "role": "user", + "content": [{"type": "text", "text": "a"}, {"type": "image_url"}, {"type": "text", "text": "b"}], + } + ] + assert first_user_text(parts) == "a\nb" + assert first_user_text([{"role": "assistant", "content": "x"}]) == "" + assert first_user_text(None) == "" + + +@pytest.mark.asyncio +async def test_first_request_binds_and_later_requests_reuse_the_binding(): + router, maker = _router() + first = {"metadata": {"program_id": "t1", "user_api_key_hash": "key-1"}} + response = await router.async_pre_routing_hook(model="oracle", request_kwargs=first, messages=_messages()) + assert response is not None and response.model == "smart" + assert ( + first["metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart" and first["metadata"][PROGRAM_ID_METADATA_KEY] == "t1" + ) + assert response.routing_decision["router_type"] == "oracle" and response.routing_decision["routed_model"] == "smart" + + later = {"metadata": {"program_id": "t1", "user_api_key_hash": "key-1"}} + again = await router.async_pre_routing_hook( + model="oracle", request_kwargs=later, messages=_messages("now run the tests") + ) + assert again is not None and again.model == "smart" + assert maker.selections == 1 + assert router.binding("t1", "key-1").api_key_hash == "key-1" + + other = await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t2"}}, messages=_messages() + ) + assert other.model == "fast" and router.programs_bound == 2 + + +@pytest.mark.asyncio +async def test_concurrent_first_requests_of_one_program_share_a_single_binding(): + class _SlowMaker(_RecordingMaker): + async def select(self, context: ProgramContext) -> str: + await asyncio.sleep(0.01) # a decision maker that does I/O (pre_routing, custom) + return await super().select(context) + + router, maker = _router(maker=_SlowMaker()) + responses = await asyncio.gather( + *( + router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t1"}}, messages=_messages() + ) + for _ in range(3) + ) + ) + assert {response.model for response in responses} == {"smart"} + assert maker.selections == 1 and router.programs_bound == 1 and router.binding("t1").model == "smart" + + +@pytest.mark.asyncio +async def test_a_non_finite_verifier_score_is_a_failed_verification(): + class _NaN: + async def verify(self, outcome: ProgramOutcome) -> float: + return float("nan") + + router, maker = _router(verifier=_NaN()) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + router.observe_request("t", cost=0.1, response_text="answer") + result = await router.complete("t") + assert result != result and router.verifications_failed == 1 and maker.updates == [] + assert router.binding("t") is None # the slot was released all the same + + +@pytest.mark.asyncio +async def test_requests_without_a_program_id_route_statelessly_and_never_learn(): + router, maker = _router() + kwargs = {"metadata": {}} + response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) + assert response.model == "smart" and PROGRAM_ID_METADATA_KEY not in kwargs["metadata"] + assert router.stateless_requests == 1 and router.binding("") is None + assert router.complete("", score=1.0) is None + + +@pytest.mark.asyncio +async def test_complete_releases_the_slot_now_and_learns_in_verification_order(): + router, maker = _router(verifier=_SlowVerifier({"t1": 0.05, "t2": 0.0})) + for program_id in ("t1", "t2"): + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages() + ) + router.observe_request("t1", cost=0.4, response_text="patch") + router.observe_request("t1", cost=0.2, response_text="tests pass") + router.observe_request("t2", cost=0.1, response_text="wrong") + t1 = router.complete("t1", score=1.0) + t2 = router.complete("t2", score=0.0) + assert router.binding("t1") is None and router.binding("t2") is None + assert router.pending_verifications == 2 + assert await t2 == 0.0 + assert await t1 == 1.0 + assert [u[0] for u in maker.updates] == ["t2", "t1"] + assert maker.updates[1] == ("t1", "smart", 1.0) + snapshot = await router.get_state_snapshot() + assert snapshot["programs_completed"] == 2 and snapshot["recent_accuracy"] == {"smart": 1.0, "fast": 0.0} + assert snapshot["recent_feedback"][-1]["cost"] == pytest.approx(0.6) + + +@pytest.mark.asyncio +async def test_a_full_verification_backlog_rejects_completion_and_keeps_the_program_bound(): + router, maker = _router(verifier=_SlowVerifier({"t1": 0.05}), max_pending_verifications=1) + for program_id in ("t1", "t2"): + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages() + ) + router.observe_request(program_id, cost=0.1, response_text="answer") + first = router.complete("t1", score=1.0) + with pytest.raises(VerificationBacklogFull, match="max_pending_verifications=1"): + router.complete("t2", score=0.0) + assert router.binding("t2") is not None and router.pending_verifications == 1 + assert router.verifications_rejected == 1 and router.programs_completed == 1 + assert await first == 1.0 + assert await router.complete("t2", score=0.0) == 0.0 + assert router.binding("t2") is None and [u[0] for u in maker.updates] == ["t1", "t2"] + assert (await router.get_state_snapshot())["verifications_rejected"] == 1 + + +@pytest.mark.asyncio +async def test_a_verification_that_outlives_its_timeout_fails_and_teaches_nothing(): + router, maker = _router(verifier=_SlowVerifier({"t": 5.0}), verification_timeout_seconds=0.01) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + router.observe_request("t", cost=0.1, response_text="answer") + task = router.complete("t", score=1.0) + assert task is not None and math.isnan(await task) + assert router.verifications_failed == 1 and maker.updates == [] + assert router.last_failure is not None and "verification: TimeoutError" in router.last_failure + assert router.pending_verifications == 0 + + +@pytest.mark.asyncio +async def test_a_program_s_message_logging_opt_out_is_kept_on_its_context(): + router, _ = _router() + await router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "private", "turn_off_message_logging": True}}, + messages=_messages(), + ) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "public"}}, messages=_messages() + ) + await router.async_pre_routing_hook( + model="oracle", + request_kwargs={ + "metadata": {"program_id": "by-header", "headers": {"litellm-enable-message-redaction": "true"}} + }, + messages=_messages(), + ) + assert router.binding("private").context.message_logging_off is True + assert router.binding("by-header").context.message_logging_off is True + assert router.binding("public").context.message_logging_off is False + + +@pytest.mark.asyncio +async def test_the_decision_maker_sees_the_caller_s_identity_and_none_of_the_request_s_other_state(): + router, maker = _router() + kwargs = { + "metadata": { + "program_id": "t", + "user_api_key_hash": "hash-1", + "user_api_key_team_id": "team-1", + "user_api_key_budget_reservation": {"amount": 1.0}, + "user_api_key_auth": {"api_key": "hash-1", "budget_reservation": {"amount": 1.0}}, + "routing_decision": {"router_type": "oracle"}, + "custom": "value", + }, + "litellm_metadata": {"user_api_key_user_id": "user-1", "_autorouter_baseline_route": "stamp"}, + } + await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"litellm_metadata": {"user_api_key_hash": "hash-2"}}, messages=_messages() + ) + bound, stateless = maker.contexts + assert bound.caller == { + "user_api_key_hash": "hash-1", + "user_api_key_team_id": "team-1", + "user_api_key_user_id": "user-1", + "user_api_key_auth": {"api_key": "hash-1"}, + "internal_call_origin": "autorouter_classifier", + } + assert budget_reservation_from_metadata(dict(bound.caller)) is None + assert stateless.caller == {"user_api_key_hash": "hash-2", "internal_call_origin": "autorouter_classifier"} + assert router.context_for("", "task", {"metadata": {"routing_decision": {}}}).caller == { + "internal_call_origin": "autorouter_classifier" + } + assert router.binding("t", "hash-1").context.caller == {} + + +@pytest.mark.asyncio +async def test_an_opt_out_on_a_later_turn_makes_the_whole_program_s_verification_private(): + seen: list[tuple[bool, bool | None]] = [] + + class _Peek: + async def verify(self, outcome: ProgramOutcome) -> float: + inherited = initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging") + seen.append((outcome.message_logging_off, inherited)) + return 1.0 + + router, _ = _router(verifier=_Peek()) + for program_id in ("public", "late-private"): + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages() + ) + router.observe_request("public", cost=0.1, response_text="a") + router.observe_request("late-private", cost=0.1, response_text="a") + router.mark_message_logging_off("late-private") + router.observe_request("late-private", cost=0.1, response_text="c") + assert router.binding("public").message_logging_off is False + assert router.binding("late-private").message_logging_off is True + assert await router.complete("public", score=1.0) == 1.0 + assert await router.complete("late-private", score=1.0) == 1.0 + assert seen == [(False, None), (True, True)] + assert initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging") is None + + +@pytest.mark.asyncio +async def test_a_responses_api_request_binds_its_program_on_the_input_text(): + router, _ = _router() + response = await router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "r"}}, + messages=None, + input=[{"role": "user", "content": [{"type": "input_text", "text": "write a limerick"}]}], + ) + assert response.messages is None and router.binding("r").context.prompt == "write a limerick" + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "s"}}, messages=None, input="plain text" + ) + assert router.binding("s").context.prompt == "plain text" + + +@pytest.mark.asyncio +async def test_complete_cost_override_and_payload_reach_the_verifier(): + seen: list[ProgramOutcome] = [] + + class _Capture: + async def verify(self, outcome: ProgramOutcome) -> float: + seen.append(outcome) + return 1.0 + + router, maker = _router(verifier=_Capture()) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages("do x") + ) + transcript = [{"role": "user", "content": "do x"}, {"role": "assistant", "content": "answer"}] + router.observe_request("t", cost=0.3, response_text="answer", messages=transcript) + await router.complete("t", cost=2.5, payload={"container": "c1"}) + assert seen[0].cost == 2.5 and seen[0].payload == {"container": "c1"} and seen[0].response_text == "answer" + assert list(seen[0].messages) == transcript + assert seen[0].prompt == "do x" and maker.updates == [("t", "smart", 1.0)] + + +@pytest.mark.asyncio +async def test_failed_verification_is_counted_and_does_not_update(): + class _Broken: + async def verify(self, outcome): + raise RuntimeError("sandbox down") + + router, maker = _router(verifier=_Broken()) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + router.observe_request("t", cost=0.1, response_text="answer") + result = await router.complete("t") + assert result != result and router.verifications_failed == 1 and maker.updates == [] + snapshot = await router.get_state_snapshot() + assert ( + snapshot["last_failure"] == "verification: RuntimeError: sandbox down" + ) # kept for the state endpoint, not logged + + +@pytest.mark.asyncio +async def test_expired_programs_are_evicted_when_the_table_is_full(): + now = [1000.0] + router, maker = _router(clock=lambda: now[0], program_ttl_seconds=10, max_programs=3) + for program_id in ("a", "b", "c"): + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages() + ) + now[0] += 4.0 + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "d"}}, messages=_messages() + ) + assert router.binding("a") is None and router.binding("b") is not None and router.binding("d") is not None + assert router.programs_evicted == 1 + + +@pytest.mark.asyncio +async def test_decision_is_stamped_in_the_proxy_internal_bucket_when_one_exists(): + router, _ = _router() + kwargs = {"metadata": {"program_id": "t1"}, "litellm_metadata": {"user_api_key_hash": "k"}} + response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) + assert response.model == "smart" + assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart" + assert CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"] + assert router.binding("t1", "k").api_key_hash == "k" + + +@pytest.mark.asyncio +async def test_the_same_program_id_under_another_key_is_another_program(): + router, maker = _router() + for key in ("key-a", "key-b"): + await router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "t1", "user_api_key_hash": key}}, + messages=_messages(), + ) + assert router.programs_bound == 2 and maker.selections == 2 + assert router.binding("t1") is None # no key: not the same program either + assert [binding.api_key_hash for binding in router.bindings_named("t1")] == ["key-a", "key-b"] + router.observe_request("t1", cost=0.1, response_text="answer", owner="key-b") + router.complete("t1", score=1.0, owner="key-b") + assert router.binding("t1", "key-a") is not None and router.binding("t1", "key-b") is None + await router.drain() + + +@pytest.mark.asyncio +async def test_completion_before_any_observed_response_is_deferred_to_the_next_one(): + router, maker = _router() + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + assert router.complete("t", score=1.0, cost=2.0, payload={"run": "r1"}) is None # nothing answered yet + binding = router.binding("t") + assert binding is not None and binding.deferred is not None and router.programs_completed == 0 + assert maker.updates == [] + router.observe_request("t", cost=0.3, response_text="answer") # the response the feedback raced + assert router.binding("t") is None and router.programs_completed == 1 + await router.drain() + assert maker.updates == [("t", "smart", 1.0)] + + +@pytest.mark.asyncio +async def test_pre_routing_hook_replaces_caller_supplied_internal_keys(): + router, _ = _router() + kwargs = { + "metadata": {"program_id": "t1", PROGRAM_ID_METADATA_KEY: "victim", CHOSEN_MODEL_METADATA_KEY: "fast"}, + "litellm_metadata": {PROGRAM_ID_METADATA_KEY: "victim"}, + } + response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) + assert PROGRAM_ID_METADATA_KEY not in kwargs["metadata"] and CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"] + assert kwargs["litellm_metadata"][PROGRAM_ID_METADATA_KEY] == "t1" + assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == response.model + stateless = {"metadata": {PROGRAM_ID_METADATA_KEY: "victim", CHOSEN_MODEL_METADATA_KEY: "fast"}} + await router.async_pre_routing_hook(model="oracle", request_kwargs=stateless, messages=_messages()) + assert PROGRAM_ID_METADATA_KEY not in stateless["metadata"] + assert stateless["metadata"][CHOSEN_MODEL_METADATA_KEY] in ("smart", "fast") diff --git a/tests/unit/router_strategy/oracle_router/test_router_dispatch.py b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py new file mode 100644 index 00000000000..96227f2f452 --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py @@ -0,0 +1,256 @@ +"""Router-level wiring of `auto_router/oracle_router`: registration, deferred init, dispatch, hooks, removal.""" + +import pytest + +import litellm +from litellm import Router +from litellm.router_strategy.oracle_router.config import PROGRAM_ID_METADATA_KEY +from litellm.router_strategy.oracle_router.decision import PreRoutingDecisionMaker, ProgramContext +from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook +from litellm.types.router import Deployment, LiteLLM_Params, RequestType + + +def _oracle(router: Router, name: str): + return router.registry().plugins("oracle")[name][0].strategy + + +def _model_list(decision_maker=None, verifier=None, extra=()): + config = {"available_models": ["smart", "fast"]} + if decision_maker: + config["decision_maker"] = decision_maker + if verifier: + config["verifier"] = verifier + return [ + { + "model_name": "oracle", + "litellm_params": {"model": "auto_router/oracle_router", "oracle_router_config": config}, + }, + {"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}}, + {"model_name": "fast", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}, + *extra, + ] + + +def _oracle_hooks(): + return litellm.logging_callback_manager.get_custom_loggers_for_type(OracleRouterPostCallHook) + + +class _CustomMaker: + def __init__(self, models): + self.models = tuple(models) + + async def select(self, context): + return self.models[-1] + + def update(self, context, model, score): + pass + + def snapshot(self): + return {"type": "custom"} + + +def test_prefix_is_classified_as_oracle_and_not_as_a_semantic_router(): + router = Router(model_list=[]) + params = LiteLLM_Params(model="auto_router/oracle_router") + assert router._is_oracle_router_deployment(litellm_params=params) is True + assert router._is_auto_router_deployment(litellm_params=params) is False + assert router._is_adaptive_router_deployment(litellm_params=params) is False + assert router._is_oracle_router_deployment(litellm_params=LiteLLM_Params(model="openai/gpt-4o")) is False + + +def test_init_registers_the_router_and_exactly_one_hook(): + router = Router(model_list=_model_list()) + assert "oracle" in router.registry().plugins("oracle") + assert _oracle(router, "oracle").models == ("smart", "fast") + hooks = [hook for hook in _oracle_hooks() if hook.oracle_router is _oracle(router, "oracle")] + assert len(hooks) == 1 + + +def test_init_without_config_raises(): + with pytest.raises(ValueError, match="oracle_router_config is required"): + Router(model_list=[{"model_name": "oracle", "litellm_params": {"model": "auto_router/oracle_router"}}]) + + +@pytest.mark.asyncio +async def test_pre_routing_hook_dispatches_to_the_oracle_router_and_binds_the_program(): + router = Router(model_list=_model_list(decision_maker={"type": "fixed", "model": "fast"})) + kwargs = {"metadata": {"program_id": "task-1"}} + response = await router.async_pre_routing_hook( + model="oracle", request_kwargs=kwargs, messages=[{"role": "user", "content": "fix the bug"}] + ) + assert response is not None and response.model == "fast" + assert response.routing_decision["router_type"] == "oracle" + assert kwargs["metadata"][PROGRAM_ID_METADATA_KEY] == "task-1" + assert _oracle(router, "oracle").binding("task-1").model == "fast" + + +def test_pre_routing_decision_maker_resolves_a_strategy_router_listed_after_it(): + quality = { + "model_name": "tiered", + "litellm_params": { + "model": "auto_router/quality_router", + "quality_router_default_model": "fast", + "quality_router_config": {"available_models": ["smart", "fast"]}, + }, + } + model_list = _model_list(decision_maker={"type": "pre_routing", "router": "tiered"}, extra=(quality,)) + model_list[1]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 3}} + model_list[2]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 1}} + router = Router(model_list=model_list) + assert isinstance(_oracle(router, "oracle").decision_maker, PreRoutingDecisionMaker) + + +def test_init_reads_adaptive_router_preferences_as_bandit_priors(): + model_list = _model_list() + model_list[1]["model_info"] = {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}} + router = Router(model_list=model_list) + maker = _oracle(router, "oracle").decision_maker + ctx = ProgramContext(program_id="p", prompt="x", request_type=RequestType.GENERAL) + assert maker.estimate(ctx, "smart") > maker.estimate(ctx, "fast") + + +def test_pre_routing_decision_maker_with_unknown_router_fails_init(): + with pytest.raises(ValueError, match="not a configured strategy router"): + Router(model_list=_model_list(decision_maker={"type": "pre_routing", "router": "ghost"})) + + +def test_hot_reload_keeps_the_existing_router_and_its_learned_state(): + router = Router(model_list=_model_list()) + before = _oracle(router, "oracle") + router.set_model_list(_model_list()) + assert _oracle(router, "oracle") is before + assert len([hook for hook in _oracle_hooks() if hook.oracle_router is before]) == 1 + + +def test_deleting_the_deployment_unregisters_the_router_and_its_hook(): + router = Router(model_list=_model_list()) + before = _oracle(router, "oracle") + router.delete_deployment(id=router.get_model_ids(model_name="oracle")[0]) + assert "oracle" not in router.registry().plugins("oracle") + assert all(hook.oracle_router is not before for hook in _oracle_hooks()) + + +def test_deployment_input_costs_prefer_litellm_params_over_model_info_and_skip_unknown_models(): + model_list = _model_list() + model_list[1]["litellm_params"]["input_cost_per_token"] = 2e-6 + model_list[1]["model_info"] = {"input_cost_per_token": 9e-6} + model_list[2]["model_info"] = {"input_cost_per_token": 1e-7} + router = Router(model_list=model_list) + assert router._deployment_input_costs(["smart", "fast", "ghost"]) == {"smart": 2e-6, "fast": 1e-7} + + +def test_deployment_adaptive_prefs_read_model_info_where_declared(): + model_list = _model_list() + model_list[1]["model_info"] = {"adaptive_router_preferences": {"quality_tier": 3, "strengths": ["code_generation"]}} + router = Router(model_list=model_list) + prefs = router._deployment_adaptive_prefs(["smart", "fast", "ghost"]) + assert set(prefs) == {"smart"} and prefs["smart"].quality_tier == 3 + assert prefs["smart"].strengths == [RequestType.CODE_GENERATION] + + +def test_resolve_strategy_router_finds_registered_strategy_routers_only(): + quality = { + "model_name": "tiered", + "litellm_params": { + "model": "auto_router/quality_router", + "quality_router_default_model": "fast", + "quality_router_config": {"available_models": ["smart", "fast"]}, + }, + } + model_list = _model_list(extra=(quality,)) + model_list[1]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 3}} + model_list[2]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 1}} + router = Router(model_list=model_list) + assert router._resolve_strategy_router("tiered") is router.registry().plugins("quality")["tiered"][0].strategy + assert router._resolve_strategy_router("smart") is None and router._resolve_strategy_router("ghost") is None + + +def test_init_oracle_router_deployment_registers_the_router_and_finalize_is_idempotent(): + router = Router(model_list=_model_list()[1:]) # the two plain deployments, no oracle router yet + deployment = Deployment( + model_name="late-oracle", + litellm_params=LiteLLM_Params( + model="auto_router/oracle_router", oracle_router_config={"available_models": ["smart", "fast"]} + ), + ) + router.init_oracle_router_deployment(deployment=deployment) + oracle = _oracle(router, "late-oracle") + assert oracle.models == ("smart", "fast") and oracle.router_name == "late-oracle" + router._sync_oracle_router_hooks() + router._sync_oracle_router_hooks() + assert len([hook for hook in _oracle_hooks() if hook.oracle_router is oracle]) == 1 + router._finalize_oracle_router_if_configured() # nothing new in model_list: the registry is untouched + assert _oracle(router, "late-oracle") is oracle + with pytest.raises(ValueError, match="oracle_router_config is required"): + router.init_oracle_router_deployment( + deployment=Deployment(model_name="bare", litellm_params=LiteLLM_Params(model="auto_router/oracle_router")) + ) + + +def test_upserting_an_oracle_deployment_builds_its_router_once_and_rebuilds_it_when_its_config_changes(): + router = Router(model_list=_model_list()[1:]) # the two plain deployments, no oracle router yet + deployment = Deployment( + model_name="late-oracle", + litellm_params=LiteLLM_Params( + model="auto_router/oracle_router", oracle_router_config={"available_models": ["smart", "fast"]} + ), + model_info={"id": "oracle-1"}, + ) + router.upsert_deployment(deployment=deployment) # a new id: nothing was on the router before + oracle = _oracle(router, "late-oracle") + assert oracle.models == ("smart", "fast") + router.upsert_deployment(deployment=deployment) # unchanged: the router and its learned state survive + assert _oracle(router, "late-oracle") is oracle + assert len([hook for hook in _oracle_hooks() if hook.oracle_router is oracle]) == 1 + router.upsert_deployment( + deployment=Deployment( + model_name="late-oracle", + litellm_params=LiteLLM_Params( + model="auto_router/oracle_router", oracle_router_config={"available_models": ["fast"]} + ), + model_info={"id": "oracle-1"}, + ) + ) + rebuilt = _oracle(router, "late-oracle") + assert rebuilt is not oracle and rebuilt.models == ("fast",) + assert [hook.oracle_router for hook in _oracle_hooks() if hook.oracle_router in (oracle, rebuilt)] == [rebuilt] + + +def test_custom_plugins_load_from_the_config_file_but_never_for_models_written_through_the_api(monkeypatch): + import subprocess + + popen_calls = [] + monkeypatch.setattr(subprocess, "Popen", lambda *args, **kwargs: popen_calls.append(args)) + from_config = Router(model_list=_model_list(decision_maker={"type": "custom", "path": f"{__name__}:_CustomMaker"})) + assert _oracle(from_config, "oracle").decision_maker.models == ("smart", "fast") + for config_key in ("decision_maker", "verifier"): + model_list = _model_list(**{config_key: {"type": "custom", "path": "subprocess:Popen"}}) + model_list[0]["model_info"] = {"db_model": True} # what the proxy sets for /model/new and /model/update + model_list[0]["litellm_params"]["oracle_router_config"]["available_models"] = ["/bin/sh", "-c", "id"] + with pytest.raises(ValueError, match="config file"): + Router(model_list=model_list) + router = Router(model_list=_model_list()[1:]) + with pytest.raises(ValueError, match="config file"): + router.upsert_deployment( + deployment=Deployment( + model_name="api-oracle", + litellm_params=LiteLLM_Params( + model="auto_router/oracle_router", + oracle_router_config={ + "available_models": ["smart", "fast"], + "verifier": {"type": "custom", "path": "subprocess:Popen"}, + }, + ), + model_info={"id": "api-oracle-1", "db_model": True}, + ) + ) + assert popen_calls == [] and "api-oracle" not in router.registry().plugins("oracle") + + +def test_strategy_router_dependencies_cover_available_models(): + from litellm.router_strategy.auto_router.auto_router_model_naming import strategy_router_dependencies + + deps = strategy_router_dependencies( + {"model": "auto_router/oracle_router", "oracle_router_config": {"available_models": ["smart", "fast"]}} + ) + assert {(dep.model_name, dep.role) for dep in deps} == {("smart", "tier"), ("fast", "tier")} diff --git a/tests/unit/router_strategy/oracle_router/test_verifier.py b/tests/unit/router_strategy/oracle_router/test_verifier.py new file mode 100644 index 00000000000..91018f2d4a0 --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_verifier.py @@ -0,0 +1,311 @@ +"""Verifiers: LiteLLM guardrails and reported scores become the number the decision maker learns from.""" + +import pytest +from fastapi import HTTPException + +from litellm.constants import REDACTED_BY_LITELLM +from litellm.exceptions import GuardrailRaisedException +from litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge import LLMAsAJudgeGuardrail +from litellm.router_strategy.oracle_router.verifier import ( + GuardrailVerifier, + ProgramOutcome, + ReportedVerifier, + Verifier, + build_verifier, + clamp_score, + recorded_status, + response_text, +) +from litellm.types.router import OracleVerifierConfig +from litellm.types.utils import Choices, Message, ModelResponse + + +def _outcome(score=None, cost=0.0, answer="done", messages=()) -> ProgramOutcome: + return ProgramOutcome( + program_id="p", model="smart", prompt="task", response_text=answer, cost=cost, messages=messages, score=score + ) + + +@pytest.mark.asyncio +async def test_reported_verifier_uses_the_reported_score_or_its_default(): + assert await ReportedVerifier().verify(_outcome(score=0.8)) == 0.8 + assert await ReportedVerifier().verify(_outcome(score=4.0)) == 1.0 + assert await ReportedVerifier(default=0.25).verify(_outcome()) == 0.25 + + +class _Guardrail: + """A guardrail that blocks the way LiteLLM's do: by raising, with no status record.""" + + def __init__(self, blocks: bool) -> None: + self.blocks = blocks + self.seen: list[dict] = [] + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + self.seen.append({"inputs": inputs, "input_type": input_type, "request_data": request_data}) + if self.blocks: + raise GuardrailRaisedException(guardrail_name="strict", message="blocked", blocked_content=True) + return inputs + + +class _RecordingGuardrail: + """A guardrail that reports through LiteLLM's status record, like llm_as_a_judge, and may also raise.""" + + def __init__(self, status: str, error: Exception | None = None, name: str = "judge") -> None: + self.status, self.error, self.name = status, error, name + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + self.seen = dict(request_data["metadata"]) # what the guardrail was handed, before it records anything + records = request_data["metadata"].setdefault("standard_logging_guardrail_information", []) + records.append({"guardrail_name": self.name, "guardrail_status": self.status}) # appended, like the real ones + if self.error is not None: + raise self.error + return inputs + + +@pytest.mark.asyncio +async def test_feedback_payload_cannot_pose_as_the_proxy_to_the_guardrail(): + judge = _RecordingGuardrail("success") + outcome = ProgramOutcome( + program_id="p", + model="smart", + prompt="task", + response_text="answer", + cost=0.1, + payload={ + "container": "c1", + "user_api_key_team_id": "someone-else", + "user_api_key": "sk-forged", + "internal_call_origin": "shadow_eval_judge", + "headers": {"litellm-disable-message-redaction": "true"}, + "routing_decision": {"router_type": "oracle"}, + }, + ) + assert await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(outcome) == 1.0 + assert judge.seen == {"container": "c1", "program_id": "p"} + + +@pytest.mark.asyncio +async def test_guardrail_verifier_refuses_a_final_answer_litellm_redacted_instead_of_judging_the_placeholder(): + judge = _Guardrail(blocks=False) + outcome = ProgramOutcome(program_id="p", model="smart", prompt="task", response_text=REDACTED_BY_LITELLM, cost=0.1) + with pytest.raises(RuntimeError, match="redacted before verification"): + await GuardrailVerifier("ok", lookup={"ok": judge}.get).verify(outcome) + stale_answer_behind_a_redacted_turn = ProgramOutcome( + program_id="p", + model="smart", + prompt="task", + response_text="an earlier public answer", + cost=0.1, + messages=[{"role": "user", "content": REDACTED_BY_LITELLM}], + ) + with pytest.raises(RuntimeError, match="redacted before verification"): + await GuardrailVerifier("ok", lookup={"ok": judge}.get).verify(stale_answer_behind_a_redacted_turn) + assert judge.seen == [] + + +@pytest.mark.asyncio +async def test_guardrail_verifier_hands_the_judge_the_transcript_and_the_feedback_payload(): + passing, blocking = _Guardrail(blocks=False), _Guardrail(blocks=True) + lookup = {"ok": passing, "strict": blocking}.get + transcript = ({"role": "user", "content": "task"}, {"role": "assistant", "content": "final answer"}) + outcome = ProgramOutcome( + program_id="p", + model="smart", + prompt="task", + response_text="final answer", + cost=0.0, + messages=transcript, + payload={"reference": "42"}, + ) + assert await GuardrailVerifier("ok", lookup=lookup).verify(outcome) == 1.0 + assert passing.seen == [ + { + "inputs": {"texts": ["final answer"], "model": "smart", "structured_messages": list(transcript)}, + "input_type": "response", + "request_data": {"model": "smart", "metadata": {"program_id": "p", "reference": "42"}}, + } + ] + assert await GuardrailVerifier("strict", lookup=lookup).verify(outcome) == 0.0 + with pytest.raises(LookupError): + await GuardrailVerifier("missing", lookup=lookup).verify(outcome) + + +@pytest.mark.asyncio +async def test_guardrail_verifier_reads_the_recorded_verdict_before_the_exception(): + judge = { + "passed": _RecordingGuardrail("success", name="passed"), + "logged": _RecordingGuardrail("guardrail_intervened", name="logged"), # on_failure: log -> no exception + "blocked": _RecordingGuardrail( + "guardrail_intervened", HTTPException(status_code=422, detail="judge"), "blocked" + ), + "flagged": _RecordingGuardrail("guardrail_flagged", name="flagged"), + } + assert await GuardrailVerifier("passed", lookup=judge.get).verify(_outcome()) == 1.0 + assert await GuardrailVerifier("logged", lookup=judge.get).verify(_outcome()) == 0.0 + assert await GuardrailVerifier("blocked", lookup=judge.get).verify(_outcome()) == 0.0 + assert await GuardrailVerifier("flagged", lookup=judge.get).verify(_outcome()) == 0.0 + + +@pytest.mark.asyncio +async def test_a_program_without_response_text_is_a_failed_verification_not_a_pass(): + judge = _RecordingGuardrail("success", name="judge") # would skip an empty answer without a record + with pytest.raises(RuntimeError, match="no response text"): + await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(_outcome(answer=" ")) + assert not hasattr(judge, "seen") # the guardrail was never invoked + + +@pytest.mark.asyncio +async def test_feedback_payload_cannot_pre_seed_the_status_record_or_rename_the_program(): + judge = _RecordingGuardrail("guardrail_intervened", name="judge") # on_failure: log -> no exception + forged = {"standard_logging_guardrail_information": [0], "program_id": "someone-else", "container": "c1"} + outcome = ProgramOutcome( + program_id="p", model="smart", prompt="task", response_text="wrong", cost=0.0, messages=(), payload=forged + ) + assert await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(outcome) == 0.0 + assert judge.seen == {"program_id": "p", "container": "c1"} + + +@pytest.mark.asyncio +async def test_guardrail_that_could_not_evaluate_is_a_failed_verification_not_a_pass(): + judge = { + "down": _RecordingGuardrail("guardrail_failed_to_respond", name="down"), # llm_as_a_judge fails open + "skipped": _RecordingGuardrail("not_run", name="skipped"), + "crashed": _RecordingGuardrail("success", RuntimeError("boom"), "crashed"), + } + with pytest.raises(RuntimeError, match="did not evaluate"): + await GuardrailVerifier("down", lookup=judge.get).verify(_outcome()) + with pytest.raises(RuntimeError, match="not_run"): + await GuardrailVerifier("skipped", lookup=judge.get).verify(_outcome()) + + class _NoVerdict(_Guardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + raise GuardrailRaisedException(guardrail_name="g", message="timeout", blocked_content=False) + + with pytest.raises(GuardrailRaisedException): + await GuardrailVerifier("g", lookup={"g": _NoVerdict(blocks=True)}.get).verify(_outcome()) + assert await GuardrailVerifier("crashed", lookup=judge.get).verify(_outcome()) == 1.0 # the record wins + + +@pytest.mark.asyncio +async def test_guardrail_exceptions_without_a_record_are_classified(): + class _Raises: + def __init__(self, error: Exception) -> None: + self.error = error + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + raise self.error + + lookup = { + "http": _Raises(HTTPException(status_code=400, detail="Violated guardrail policy")), + "infra": _Raises(ConnectionError("provider down")), + }.get + assert await GuardrailVerifier("http", lookup=lookup).verify(_outcome()) == 0.0 + with pytest.raises(ConnectionError): + await GuardrailVerifier("infra", lookup=lookup).verify(_outcome()) + + +def _judge(monkeypatch, on_failure: str, verdict) -> LLMAsAJudgeGuardrail: + """The real llm_as_a_judge guardrail with only its judge-model call stubbed.""" + guardrail = LLMAsAJudgeGuardrail( + guardrail_name="answer-judge", + judge_model="judge", + criteria=[{"name": "task_completed", "weight": 100, "description": "the task was completed"}], + overall_threshold=50, + on_failure=on_failure, + ) + + async def run_judge(messages, text_under_review, input_type="response"): + if isinstance(verdict, Exception): + raise verdict + return verdict + + monkeypatch.setattr(guardrail, "_run_judge", run_judge) + return guardrail + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("on_failure", "overall_score", "expected"), + [ + ("block", 90, 1.0), # passes the threshold: recorded as success + ("block", 10, 0.0), # fails: the judge raises 422 and records guardrail_intervened + ("log", 10, 0.0), # fails but only logs: no exception, still recorded as guardrail_intervened + ], +) +async def test_real_llm_as_a_judge_verdicts_become_scores(monkeypatch, on_failure, overall_score, expected): + guardrail = _judge(monkeypatch, on_failure, {"overall_score": overall_score, "verdicts": []}) + verifier = GuardrailVerifier("answer-judge", lookup={"answer-judge": guardrail}.get) + assert await verifier.verify(_outcome(answer="final answer")) == expected + + +@pytest.mark.asyncio +async def test_real_llm_as_a_judge_that_fails_open_is_not_a_pass(monkeypatch): + guardrail = _judge(monkeypatch, "block", RuntimeError("judge model unreachable")) # the guardrail returns normally + verifier = GuardrailVerifier("answer-judge", lookup={"answer-judge": guardrail}.get) + with pytest.raises(RuntimeError, match="did not evaluate"): + await verifier.verify(_outcome(answer="final answer")) + + +def test_recorded_status_reads_both_buckets_and_takes_the_latest(): + data = { + "metadata": { + "standard_logging_guardrail_information": [{"guardrail_name": "a", "guardrail_status": "success"}] + }, + "litellm_metadata": { + "standard_logging_guardrail_information": {"guardrail_name": "a", "guardrail_status": "guardrail_flagged"} + }, + } + assert recorded_status(data, "a") == "success" # a single dict, not a list, is not a valid record set + data["litellm_metadata"]["standard_logging_guardrail_information"] = [ + {"guardrail_name": "a", "guardrail_status": "guardrail_intervened"} + ] + assert recorded_status(data, "a") == "guardrail_intervened" + assert recorded_status(data, "other") is None + assert recorded_status({"metadata": "junk"}, "a") is None + + +def test_clamp_score_rejects_non_finite_scores(): + assert clamp_score(1.7) == 1.0 and clamp_score(-2) == 0.0 + for bad in (float("nan"), float("inf")): + with pytest.raises(ValueError, match="finite"): + clamp_score(bad) + + +@pytest.mark.asyncio +async def test_reported_verifier_rejects_a_nan_score(): + with pytest.raises(ValueError, match="finite"): + await ReportedVerifier().verify(_outcome(score=float("nan"))) + + +@pytest.mark.asyncio +async def test_guardrail_verifier_without_a_transcript_sends_the_initial_request(): + passing = _Guardrail(blocks=False) + await GuardrailVerifier("ok", lookup={"ok": passing}.get).verify(_outcome()) + assert passing.seen[0]["inputs"]["structured_messages"] == [{"role": "user", "content": "task"}] + + +def test_response_text_reads_chat_completions_only(): + assert response_text(ModelResponse(choices=[Choices(message=Message(content="hi", role="assistant"))])) == "hi" + assert response_text({"choices": []}) == "" + assert response_text(None) == "" + + +class CustomVerifier: + async def verify(self, outcome: ProgramOutcome) -> float: + return 0.5 + + +class NotAVerifier: + pass + + +@pytest.mark.asyncio +async def test_build_verifier_by_type(): + assert isinstance(build_verifier(OracleVerifierConfig(type="reported")), ReportedVerifier) + assert isinstance(build_verifier(OracleVerifierConfig(type="guardrail", guardrail="g")), GuardrailVerifier) + custom = build_verifier(OracleVerifierConfig(type="custom", path=f"{__name__}:CustomVerifier")) + assert isinstance(custom, Verifier) and await custom.verify(_outcome()) == 0.5 + with pytest.raises(TypeError): + build_verifier(OracleVerifierConfig(type="custom", path=f"{__name__}:NotAVerifier")) + with pytest.raises(ValueError, match=r"verifier\.guardrail"): + OracleVerifierConfig(type="guardrail") diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 15d440c0ad4..65f92affcea 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11651,6 +11651,51 @@ export interface paths { patch: operations["openrouter_proxy_route_openrouter__endpoint__patch"]; trace?: never; }; + "/oracle_router/feedback": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Submit Oracle Router Feedback + * @description Complete a program: its model slot is released now, its verifier runs in the background. + * + * Programs are private to the key that started them; an admin may complete any key's program. 404 when + * the caller holds no program with that id, which also covers programs completed earlier. A program none + * of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed + * response completes it with this feedback. + */ + post: operations["submit_oracle_router_feedback_oracle_router_feedback_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/oracle_router/state": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Oracle Router State + * @description Decision-maker state, active bindings and recent verified feedback for every ORACLE router. Admin only. + */ + get: operations["get_oracle_router_state_oracle_router_state_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/organization/daily/activity": { parameters: { query?: never; @@ -36100,6 +36145,10 @@ export interface components { openai_identity_token_file?: string | null; /** Openai Service Account Id */ openai_service_account_id?: string | null; + /** Oracle Router Config */ + oracle_router_config?: { + [key: string]: unknown; + } | null; /** Organization */ organization?: string | null; /** Otpm */ @@ -40652,6 +40701,52 @@ export interface components { } & { [key: string]: unknown; }; + /** + * OracleRouterFeedbackRequest + * @description Report that a program finished. ``score`` is optional when the router's verifier grades it. + */ + OracleRouterFeedbackRequest: { + /** + * Cost + * @description Overrides the spend LiteLLM accumulated for the program + */ + cost?: number | null; + /** + * Payload + * @description Passed to the verifier unchanged + */ + payload?: { + [key: string]: unknown; + }; + /** Program Id */ + program_id: string; + /** Score */ + score?: number | null; + }; + /** OracleRouterFeedbackResponse */ + OracleRouterFeedbackResponse: { + /** + * Deferred + * @description No response of the program has been observed yet; its next one completes it with this feedback + * @default false + */ + deferred: boolean; + /** Model */ + model: string; + /** Pending Verifications */ + pending_verifications: number; + /** Program Id */ + program_id: string; + /** Router Name */ + router_name: string; + }; + /** OracleRouterStateResponse */ + OracleRouterStateResponse: { + /** Routers */ + routers: { + [key: string]: unknown; + }[]; + }; /** OrgMember */ OrgMember: { /** @@ -46637,7 +46732,7 @@ export interface components { * Router Type * @enum {string} */ - router_type?: "complexity" | "adaptive" | "quality" | "semantic"; + router_type?: "complexity" | "adaptive" | "quality" | "semantic" | "oracle"; /** Savings Baseline Deployment Id */ savings_baseline_deployment_id?: string; /** Savings Baseline Model */ @@ -51783,6 +51878,10 @@ export interface components { openai_identity_token_file?: string | null; /** Openai Service Account Id */ openai_service_account_id?: string | null; + /** Oracle Router Config */ + oracle_router_config?: { + [key: string]: unknown; + } | null; /** Organization */ organization?: string | null; /** Otpm */ @@ -68875,6 +68974,59 @@ export interface operations { }; }; }; + submit_oracle_router_feedback_oracle_router_feedback_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["OracleRouterFeedbackRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["OracleRouterFeedbackResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_oracle_router_state_oracle_router_state_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["OracleRouterStateResponse"]; + }; + }; + }; + }; get_organization_daily_activity_organization_daily_activity_get: { parameters: { query?: {