From b02990c9c96471971fb8387b020a3f8487d57448 Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Fri, 2 Oct 2026 15:39:04 -0400 Subject: [PATCH 01/10] feat(router): add ORACLE program-level router with pluggable decision makers and verifiers Adds auto_router/oracle_router, a pre-routing strategy that binds each agentic program (all requests sharing a program_id) to one model at its first request and learns from the program's verified outcome after it finishes. Decision makers: linucb (default), ucb, epsilon_greedy, fixed, pre_routing (wraps a complexity, quality, adaptive or semantic router) and custom import path. Verifiers: reported score, any configured guardrail, an LLM judge through the proxy's router, and custom import path. Reward combines the score with the spend LiteLLM computed for the program. Endpoints: POST /oracle_router/feedback completes a program, GET /oracle_router/state shows bindings and learned state. Response header x-litellm-oracle-router-model names the bound model. Paper: https://arxiv.org/abs/2607.22465 --- .../oracle_router_example.yaml | 51 +++ .../model_management_endpoints.py | 1 + .../oracle_router_endpoints.py | 94 +++++ litellm/proxy/proxy_server.py | 4 + litellm/router.py | 150 +++++++- .../router_strategy/oracle_router/README.md | 54 +++ .../router_strategy/oracle_router/__init__.py | 32 ++ .../router_strategy/oracle_router/config.py | 18 + .../router_strategy/oracle_router/decision.py | 232 ++++++++++++ .../router_strategy/oracle_router/hooks.py | 88 +++++ .../oracle_router/oracle_router.py | 357 ++++++++++++++++++ .../router_strategy/oracle_router/verifier.py | 120 ++++++ .../router_utils/auto_router_model_naming.py | 8 +- .../oracle_router_endpoints.py | 27 ++ litellm/types/router.py | 76 +++- litellm/types/utils.py | 2 +- .../test_oracle_router_endpoints.py | 124 ++++++ .../router_strategy/oracle_router/__init__.py | 0 .../oracle_router/test_decision.py | 171 +++++++++ .../oracle_router/test_hooks.py | 119 ++++++ .../oracle_router/test_oracle_router.py | 206 ++++++++++ .../oracle_router/test_router_dispatch.py | 125 ++++++ .../oracle_router/test_verifier.py | 100 +++++ 23 files changed, 2154 insertions(+), 5 deletions(-) create mode 100644 litellm/proxy/example_config_yaml/oracle_router_example.yaml create mode 100644 litellm/proxy/management_endpoints/oracle_router_endpoints.py create mode 100644 litellm/router_strategy/oracle_router/README.md create mode 100644 litellm/router_strategy/oracle_router/__init__.py create mode 100644 litellm/router_strategy/oracle_router/config.py create mode 100644 litellm/router_strategy/oracle_router/decision.py create mode 100644 litellm/router_strategy/oracle_router/hooks.py create mode 100644 litellm/router_strategy/oracle_router/oracle_router.py create mode 100644 litellm/router_strategy/oracle_router/verifier.py create mode 100644 litellm/types/management_endpoints/oracle_router_endpoints.py create mode 100644 tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py create mode 100644 tests/unit/router_strategy/oracle_router/__init__.py create mode 100644 tests/unit/router_strategy/oracle_router/test_decision.py create mode 100644 tests/unit/router_strategy/oracle_router/test_hooks.py create mode 100644 tests/unit/router_strategy/oracle_router/test_oracle_router.py create mode 100644 tests/unit/router_strategy/oracle_router/test_router_dispatch.py create mode 100644 tests/unit/router_strategy/oracle_router/test_verifier.py diff --git a/litellm/proxy/example_config_yaml/oracle_router_example.yaml b/litellm/proxy/example_config_yaml/oracle_router_example.yaml new file mode 100644 index 00000000000..f8d64bde8e1 --- /dev/null +++ b/litellm/proxy/example_config_yaml/oracle_router_example.yaml @@ -0,0 +1,51 @@ +# ORACLE router: one model per agentic program, learned from verified outcomes. +# +# Clients call `model: oracle-router` and tag every request of a task with the same program id: +# POST /v1/chat/completions { "model": "oracle-router", "metadata": { "program_id": "task-17" }, ... } +# The first request binds the program to a model; later requests reuse it. On the last request add +# "metadata": { "program_id": "task-17", "program_done": true, "program_score": 1.0 } +# or POST /oracle_router/feedback { "program_id": "task-17", "score": 1.0 } once the task is graded. +# +# Required env vars: OPENAI_API_KEY, LITELLM_MASTER_KEY. + +model_list: + - model_name: oracle-router + litellm_params: + model: auto_router/oracle_router + oracle_router_config: + available_models: ["smart", "fast"] # strongest first + decision_maker: + type: thompson # fixed | pre_routing | custom + weights: { quality: 0.7, cost: 0.3 } # the adaptive router's quality/cost trade-off + verifier: + type: guardrail # reported | custom + guardrail: answer-judge + + # Let LiteLLM's complexity router propose the model and have ORACLE bind, verify and report it: + # decision_maker: { type: pre_routing, router: tiered-router } + + - model_name: smart + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + + - model_name: fast + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +guardrails: + - guardrail_name: answer-judge + litellm_params: + guardrail: llm_as_a_judge + mode: post_call + default_on: false + judge_model: smart + overall_threshold: 50 + criteria: + - name: task_completed + weight: 100 + description: The assistant completed the user's task correctly, judged from the whole conversation. + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 50bb831d169..0596946f44e 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -3322,6 +3322,7 @@ async def clear_cache() -> ReconcileOutcome: llm_router.complexity_routers.pop(model_name, None) llm_router.adaptive_routers.pop(model_name, None) llm_router.quality_routers.pop(model_name, None) + llm_router.oracle_routers.pop(model_name, None) # Reload only DB models. _add_deployment_locked, not add_deployment: this # coroutine already holds MODEL_RECONCILE_LOCK and asyncio.Lock is not diff --git a/litellm/proxy/management_endpoints/oracle_router_endpoints.py b/litellm/proxy/management_endpoints/oracle_router_endpoints.py new file mode 100644 index 00000000000..27e79fe6785 --- /dev/null +++ b/litellm/proxy/management_endpoints/oracle_router_endpoints.py @@ -0,0 +1,94 @@ +""" +ORACLE ROUTER MANAGEMENT ENDPOINTS + +POST /oracle_router/feedback - Report a finished program so its verifier runs and the router learns +GET /oracle_router/state - Live decision-maker state, bindings and recent feedback per router +""" + +from itertools import chain +from typing import TYPE_CHECKING, Annotated, Final + +from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.types.management_endpoints.oracle_router_endpoints import ( + OracleRouterFeedbackRequest, + OracleRouterFeedbackResponse, + OracleRouterStateResponse, +) + +if TYPE_CHECKING: + from fastapi import APIRouter, Depends, HTTPException + + from litellm.router_strategy.oracle_router.oracle_router import OracleRouter +else: + try: + from fastapi import APIRouter, Depends, HTTPException + except ImportError: + pass + +router: Final = APIRouter() + + +def _is_admin(user_api_key_dict: UserAPIKeyAuth) -> bool: + return user_api_key_dict.user_role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + + +def _oracle_routers() -> tuple["OracleRouter", ...]: + from litellm.proxy.proxy_server import llm_router + + registry: Final = llm_router.oracle_routers if llm_router is not None else {} + tagged_entries: Final = tuple(chain.from_iterable(registry.values())) + return tuple(tagged.strategy for tagged in tagged_entries) + + +@router.post( + "/oracle_router/feedback", + tags=["oracle_router"], + dependencies=[Depends(user_api_key_auth)], + response_model=OracleRouterFeedbackResponse, +) +async def submit_oracle_router_feedback( + data: OracleRouterFeedbackRequest, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> OracleRouterFeedbackResponse: + """Complete a program: its model slot is released now, its verifier runs in the background. + + The key that started the program (or an admin) may complete it. 404 when no ORACLE router holds a + program with that id, which also covers programs completed earlier. + """ + routers: Final = _oracle_routers() + if not routers: + raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."}) + owner: Final = next((candidate for candidate in routers if candidate.binding(data.program_id) is not None), None) + binding: Final = owner.binding(data.program_id) if owner is not None else None + if owner is None or binding is None: + raise HTTPException( + status_code=404, detail={"error": f"Unknown or already completed program {data.program_id!r}."} + ) + if binding.api_key_hash and binding.api_key_hash != user_api_key_dict.api_key and not _is_admin(user_api_key_dict): + raise HTTPException(status_code=403, detail={"error": CommonProxyErrors.not_allowed_access.value}) + owner.complete(data.program_id, score=data.score, cost=data.cost, payload=data.payload) + return OracleRouterFeedbackResponse( + program_id=data.program_id, + router_name=owner.router_name, + model=binding.model, + pending_verifications=owner.pending_verifications, + ) + + +@router.get( + "/oracle_router/state", + tags=["oracle_router"], + dependencies=[Depends(user_api_key_auth)], + response_model=OracleRouterStateResponse, +) +async def get_oracle_router_state( + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +) -> OracleRouterStateResponse: + """Decision-maker state, active bindings and recent verified feedback for every ORACLE router. Admin only.""" + if not _is_admin(user_api_key_dict): + raise HTTPException(status_code=403, detail={"error": CommonProxyErrors.not_allowed_access.value}) + routers: Final = _oracle_routers() + if not routers: + raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."}) + return OracleRouterStateResponse(routers=[await candidate.get_state_snapshot() for candidate in routers]) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index e406e6faec4..baa59772ffd 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -643,6 +643,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import ( from litellm.proxy.management_endpoints.model_management_endpoints import ( router as model_management_router, ) +from litellm.proxy.management_endpoints.oracle_router_endpoints import ( + router as oracle_router_management_router, +) from litellm.proxy.management_endpoints.organization_endpoints import ( router as organization_router, ) @@ -19995,6 +19998,7 @@ app.include_router(budget_management_router) app.include_router(model_management_router) app.include_router(model_access_group_management_router) app.include_router(auto_router_management_router) +app.include_router(oracle_router_management_router) app.include_router(tag_management_router) app.include_router(workflow_management_router) app.include_router(memory_router) diff --git a/litellm/router.py b/litellm/router.py index 24554e61516..9af3fc2fab6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -273,6 +273,7 @@ from litellm.types.llms.openai import ( from litellm.types.router import ( CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS, VALID_LITELLM_ENVIRONMENTS, + AdaptiveRouterPreferences, AlertingConfig, AllowedFailsPolicy, AssistantsTypedDict, @@ -357,6 +358,7 @@ if TYPE_CHECKING: from litellm.router_strategy.complexity_router.complexity_router import ( ComplexityRouter, ) + from litellm.router_strategy.oracle_router.oracle_router import OracleRouter from litellm.router_strategy.quality_router.quality_router import ( QualityRouter, ) @@ -377,6 +379,7 @@ else: ComplexityRouter = Any AdaptiveRouter = Any QualityRouter = Any + OracleRouter = Any # rebind-ok: runtime stand-in for the TYPE_CHECKING-only import above PreRoutingHookResponse = Any RouterStrategySelector: TypeAlias = ( @@ -996,6 +999,7 @@ class Router: self.complexity_routers: dict[str, list[TaggedPreRoutingStrategy[ComplexityRouter]]] = {} self.adaptive_routers: dict[str, list[TaggedPreRoutingStrategy[AdaptiveRouter]]] = {} self.quality_routers: dict[str, list[TaggedPreRoutingStrategy[QualityRouter]]] = {} + self.oracle_routers: dict[str, list[TaggedPreRoutingStrategy[OracleRouter]]] = {} # mutable-ok: registry self.routing_plugins: list[RoutingPlugin] = list(plugins) if plugins else [] # Initialize model_group_alias early since it's used in set_model_list @@ -9307,6 +9311,8 @@ class Router: self._unregister_pre_routing_strategy(registry, model_name, tags) if self._unregister_pre_routing_strategy(self.adaptive_routers, model_name, tags): self._sync_adaptive_router_hooks() + if self._unregister_pre_routing_strategy(self.oracle_routers, model_name, tags): + self._sync_oracle_router_hooks() def _finalize_adaptive_router_if_configured(self) -> None: """Locate every adaptive-router deployment in the finalized model_list and @@ -9436,6 +9442,129 @@ class Router: len(config.available_models), ) + def _is_oracle_router_deployment(self, litellm_params: LiteLLM_Params) -> bool: + """True when this deployment opts in via the `auto_router/oracle_router` model prefix.""" + return classify_strategy_router_model(litellm_params.model) == "oracle" + + def _finalize_oracle_router_if_configured(self) -> None: + """Build an OracleRouter for every oracle-router deployment in the finalized model_list. + + Deferred like the adaptive router: a `pre_routing` decision maker wraps another strategy + router that may appear later in the list. Idempotent across hot reloads, so a router that + already holds program bindings and learned state is never rebuilt.""" + for entry in self.model_list or []: + lp = entry.get("litellm_params") if isinstance(entry, dict) else entry.litellm_params + lp_model = (lp.get("model") if isinstance(lp, dict) else lp.model) if lp else None + if not (lp_model and classify_strategy_router_model(lp_model) == "oracle"): + continue + model_name = entry.get("model_name") if isinstance(entry, dict) else entry.model_name + if not model_name or not lp: + continue + deployment = Deployment( + model_name=model_name, + litellm_params=(lp if not isinstance(lp, dict) else LiteLLM_Params(**lp)), + model_info=(entry.get("model_info") if isinstance(entry, dict) else entry.model_info), + ) + if self._has_registered_strategy(self.oracle_routers, model_name, self._deployment_tags(deployment)): + continue + self.init_oracle_router_deployment(deployment=deployment) + self._sync_oracle_router_hooks() + + def _sync_oracle_router_hooks(self) -> None: + """Exactly one post-call hook per registered oracle router.""" + from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook + + for callback in litellm.logging_callback_manager.get_custom_loggers_for_type(OracleRouterPostCallHook): + litellm.logging_callback_manager.remove_callback_from_all_lists(callback) + for tagged_oracle_routers in self.oracle_routers.values(): + for tagged in tagged_oracle_routers: + litellm.logging_callback_manager.add_litellm_callback( + OracleRouterPostCallHook(oracle_router=tagged.strategy) + ) + + def _deployment_input_costs( + self, model_names: Sequence[str] + ) -> dict[str, float]: # mutable-ok: pick_best takes a dict + """``input_cost_per_token`` of the first deployment of each model name, from litellm_params or model_info.""" + costs: Final[dict[str, float]] = {} # mutable-ok: built per model name in the loop below + for name in model_names: + indices = self.model_name_to_deployment_indices.get(name, []) + if not indices: + continue + entry = (self.model_list or [])[indices[0]] + lp = entry.get("litellm_params") if isinstance(entry, dict) else entry.litellm_params + lp_dict = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) + mi = entry.get("model_info") if isinstance(entry, dict) else entry.model_info + mi_dict = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) + cost = lp_dict.get("input_cost_per_token") + if cost is None: + cost = mi_dict.get("input_cost_per_token") + if cost is not None: + costs[name] = float(cost) + return costs + + def _deployment_adaptive_prefs( + self, model_names: Sequence[str] + ) -> dict[str, AdaptiveRouterPreferences]: # mutable-ok: consumed as a mapping by the bandit + """``model_info.adaptive_router_preferences`` of the first deployment of each model name, where declared.""" + prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} # mutable-ok: built per model name in the loop below + for name in model_names: + indices = self.model_name_to_deployment_indices.get(name, []) + if not indices: + continue + entry = (self.model_list or [])[indices[0]] + mi = entry.get("model_info") if isinstance(entry, dict) else entry.model_info + mi_dict = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) + raw = mi_dict.get("adaptive_router_preferences") + if raw is not None: + prefs[name] = AdaptiveRouterPreferences(**raw) + return prefs + + def _resolve_strategy_router(self, model_name: str) -> "PreRoutingStrategy | None": + """The first strategy router registered under `model_name`, for ORACLE's `pre_routing` decision maker.""" + for registry in (self.complexity_routers, self.quality_routers, self.adaptive_routers, self.auto_routers): + tagged_routers = registry.get(model_name) + if tagged_routers: + return tagged_routers[0].strategy + return None + + def init_oracle_router_deployment(self, deployment: Deployment) -> None: + """Build an OracleRouter from `oracle_router_config` and register it under `deployment.model_name`.""" + from litellm.router_strategy.oracle_router.decision import build_decision_maker + from litellm.router_strategy.oracle_router.oracle_router import OracleRouter + from litellm.router_strategy.oracle_router.verifier import build_verifier + from litellm.types.router import OracleRouterConfig + + raw_config: Final = deployment.litellm_params.oracle_router_config + if raw_config is None: + raise ValueError("oracle_router_config is required for oracle-router deployments.") + config: Final = OracleRouterConfig(**raw_config) + oracle_router: Final = OracleRouter( + router_name=deployment.model_name, + config=config, + decision_maker=build_decision_maker( + config.decision_maker, + tuple(config.available_models), + self._deployment_input_costs(config.available_models), + self._resolve_strategy_router, + self._deployment_adaptive_prefs(config.available_models), + ), + verifier=build_verifier(config.verifier), + ) + self._register_pre_routing_strategy( + registry=self.oracle_routers, + deployment=deployment, + strategy=oracle_router, + strategy_label="Oracle-router", + ) + verbose_router_logger.info( + "OracleRouter[%s] initialized with %d models, decision_maker=%s, verifier=%s", + deployment.model_name, + len(config.available_models), + config.decision_maker.type, + config.verifier.type, + ) + def _is_quality_router_deployment(self, litellm_params: LiteLLM_Params) -> bool: """ Check if the deployment is a quality-router deployment. @@ -9589,6 +9718,7 @@ class Router: # Deferred: build the AdaptiveRouter strategy now that all underlying # deployments have been registered. self._finalize_adaptive_router_if_configured() + self._finalize_oracle_router_if_configured() def _add_deployment(self, deployment: Deployment) -> Deployment: import os @@ -9958,6 +10088,8 @@ class Router: ) ): self._finalize_adaptive_router_if_configured() + if self._is_oracle_router_deployment(litellm_params=_deployment_on_router.litellm_params): + self._finalize_oracle_router_if_configured() return deployment except Exception as e: if self.ignore_invalid_deployments: @@ -13649,7 +13781,13 @@ class Router: deployments: returning None hands the request to ordinary tag-aware deployment selection. """ - registries: Final = (self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers) + registries: Final = ( + self.auto_routers, + self.complexity_routers, + self.adaptive_routers, + self.quality_routers, + self.oracle_routers, + ) if not any(registries): return None deployments: Final = self.deployments_for_request(model, request_kwargs) @@ -13741,7 +13879,15 @@ class Router: ) -> str: if is_native_compaction_call(): return registered_model_name - if not any((self.auto_routers, self.complexity_routers, self.adaptive_routers, self.quality_routers)): + if not any( + ( + self.auto_routers, + self.complexity_routers, + self.adaptive_routers, + self.quality_routers, + self.oracle_routers, + ) + ): return registered_model_name cache_key: Final = self._claude_code_session_router_cache_key(request_kwargs) if cache_key is None or not isinstance(request_kwargs, dict): diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md new file mode 100644 index 00000000000..7827e735323 --- /dev/null +++ b/litellm/router_strategy/oracle_router/README.md @@ -0,0 +1,54 @@ +# ORACLE router + +Program-level routing for agentic workloads: an agentic task (a *program*) is bound to one model at its +first request, every later request with the same `program_id` reuses that binding, and the model is +chosen by a pluggable decision maker that learns from a pluggable verifier's delayed verdict. +Paper: https://arxiv.org/abs/2607.22465 + +## Config + +```yaml +model_list: + - model_name: oracle-router + litellm_params: + model: auto_router/oracle_router + oracle_router_config: + available_models: ["smart", "fast"] + decision_maker: { type: thompson, weights: { quality: 0.7, cost: 0.3 } } + verifier: { type: guardrail, guardrail: answer-judge } +``` + +Clients send `metadata.program_id` (or `litellm_session_id`, or the `x-litellm-program-id` header) +with every request of a task. Requests without any id are routed statelessly and never learn. + +## Decision makers + +| `type` | binds a program by | learns | +|---|---|---| +| `thompson` (default) | LiteLLM's adaptive-router bandit: a Beta posterior per (request type, model), Thompson sampling, cost-weighted pick with `weights: {quality, cost}` | from verified program scores | +| `pre_routing` | the strategy router named by `router` (complexity, quality, adaptive or semantic), once per program | no | +| `fixed` | always `model` | no | +| `custom` | `path: pkg.mod:Factory`, called with the model tuple; must expose `models`, `async select`, `update`, `snapshot` | up to you | + +## Verifiers + +| `type` | scores a finished program by | +|---|---| +| `guardrail` | applying the LiteLLM guardrail named `guardrail` to the program's final turn, with the transcript as `structured_messages` and the feedback `payload` as `request_data["metadata"]`: pass = 1, blocked = 0. An `llm_as_a_judge` guardrail is the usual choice | +| `reported` (default) | `metadata.program_score` on the last request, or `POST /oracle_router/feedback` | +| `custom` | `path: pkg.mod:Factory`, called with no arguments; must expose `async verify(outcome)` | + +A verified score `s` adds `s` successes and `1 - s` failures to the cell of the model that served the +program. Updates are applied in verification-arrival order. + +## Finishing a program + +Either mark the last request with `metadata.program_done: true` (plus `program_score` for the reported +verifier), or call `POST /oracle_router/feedback {"program_id": ..., "score": ..., "cost": ...}` from +the harness once the task is graded. `GET /oracle_router/state` (admin) shows bindings, decision-maker +state and recent feedback. The response header `x-litellm-oracle-router-model` names the bound model. + +## Limits + +State is in memory per proxy process; it is not persisted or shared across replicas. The paper's +dispatch scheduler (DISC) is not part of this strategy. diff --git a/litellm/router_strategy/oracle_router/__init__.py b/litellm/router_strategy/oracle_router/__init__.py new file mode 100644 index 00000000000..d5bc771d923 --- /dev/null +++ b/litellm/router_strategy/oracle_router/__init__.py @@ -0,0 +1,32 @@ +from litellm.router_strategy.oracle_router.decision import ( + DecisionMaker, + FixedDecisionMaker, + PreRoutingDecisionMaker, + ProgramContext, + ThompsonDecisionMaker, + build_decision_maker, +) +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, ProgramBinding +from litellm.router_strategy.oracle_router.verifier import ( + GuardrailVerifier, + ProgramOutcome, + ReportedVerifier, + Verifier, + build_verifier, +) + +__all__ = [ + "DecisionMaker", + "FixedDecisionMaker", + "GuardrailVerifier", + "OracleRouter", + "PreRoutingDecisionMaker", + "ProgramBinding", + "ProgramContext", + "ProgramOutcome", + "ReportedVerifier", + "ThompsonDecisionMaker", + "Verifier", + "build_decision_maker", + "build_verifier", +] diff --git a/litellm/router_strategy/oracle_router/config.py b/litellm/router_strategy/oracle_router/config.py new file mode 100644 index 00000000000..a7ed8ccc1e7 --- /dev/null +++ b/litellm/router_strategy/oracle_router/config.py @@ -0,0 +1,18 @@ +"""Metadata keys and headers shared by the ORACLE router, its post-call hook and the proxy endpoints.""" + +from typing import Final + +ORACLE_ROUTER_MODEL_PREFIX: Final[str] = "auto_router/oracle_router" + +CHOSEN_MODEL_METADATA_KEY: Final[str] = "oracle_router_chosen_model" +PROGRAM_ID_METADATA_KEY: Final[str] = "oracle_router_program_id" +RESPONSE_HEADER: Final[str] = "x-litellm-oracle-router-model" +PROGRAM_ID_HEADER: Final[str] = "x-litellm-program-id" + +PROGRAM_DONE_KEY: Final[str] = "program_done" +PROGRAM_SCORE_KEY: Final[str] = "program_score" + +SESSION_ID_FALLBACK_KEYS: Final[tuple[str, ...]] = ("litellm_session_id", "session_id") + +FEEDBACK_HISTORY_SIZE: Final[int] = 200 +PROGRAM_SWEEP_THRESHOLD: Final[int] = 1024 diff --git a/litellm/router_strategy/oracle_router/decision.py b/litellm/router_strategy/oracle_router/decision.py new file mode 100644 index 00000000000..7a6ea12e72c --- /dev/null +++ b/litellm/router_strategy/oracle_router/decision.py @@ -0,0 +1,232 @@ +"""Decision makers: the pluggable policy that binds a program to a model. + +``DecisionMaker`` is the whole contract. The default, ``thompson``, is LiteLLM's own adaptive-router +bandit (a Beta posterior per request type and model, Thompson sampling, cost-weighted pick) trained on +verified program outcomes instead of per-turn regex signals. ``pre_routing`` lets any LiteLLM strategy +router (complexity, quality, adaptive, semantic) make the proposal once per program, ``fixed`` pins one +model, and ``custom`` imports an operator's class. +""" + +import importlib +import itertools +import random +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from typing import Final, Protocol, runtime_checkable + +from litellm.router_strategy.adaptive_router.bandit import BanditCell, apply_delta, initial_cell, pick_best +from litellm.types.router import ( + AdaptiveRouterPreferences, + AdaptiveRouterWeights, + OracleDecisionMakerConfig, + PreRoutingStrategy, + RequestType, +) + + +@dataclass(frozen=True, slots=True) +class ProgramContext: + """What a decision maker sees when a program arrives: its initial request and LiteLLM's request type.""" + + program_id: str + prompt: str + request_type: RequestType + + +@runtime_checkable +class DecisionMaker(Protocol): + @property + def models(self) -> tuple[str, ...]: ... + + async def select(self, context: ProgramContext) -> str: ... + + def update(self, context: ProgramContext, model: str, score: float) -> None: ... + + def snapshot(self) -> Mapping[str, object]: ... + + +def _check_model(models: Sequence[str], model: str) -> None: + if model not in models: + raise KeyError(f"unknown model {model!r}; configured models: {list(models)}") + + +_UNIFORM_PRIOR: Final = AdaptiveRouterPreferences(quality_tier=2, strengths=[]) + + +class ThompsonDecisionMaker: + """LiteLLM's adaptive-router bandit, keyed by (request type, model) and updated with verified scores. + + Cells start from the same priors the adaptive router uses, read from each model's + ``model_info.adaptive_router_preferences`` (tier 2 when a model declares none). A verified score ``s`` + in [0, 1] adds ``s`` successes and ``1 - s`` failures to the cell of the model that served the program. + Selection samples every cell once and picks the best quality/cost score with the adaptive router's + weights, so ``weights.cost`` is the only cost knob. + """ + + def __init__( + self, + models: Sequence[str], + model_costs: Mapping[str, float], + weights: AdaptiveRouterWeights | None = None, + model_prefs: Mapping[str, AdaptiveRouterPreferences] | None = None, + seed: int = 0, + ) -> None: + if not models: + raise ValueError("a decision maker needs at least one model") + self._models: Final[tuple[str, ...]] = tuple(models) + costs: Final = {m: float(model_costs.get(m, 0.0)) for m in self._models} + self._costs: Final[dict[str, float]] = costs # mutable-ok: pick_best takes a dict + self._weights: Final[AdaptiveRouterWeights] = weights or AdaptiveRouterWeights() + self._rng: Final = random.Random(seed) + prefs: Final = model_prefs or {} + grid: Final = tuple(itertools.product(RequestType, self._models)) + cells: Final = {(rt, model): initial_cell(prefs.get(model, _UNIFORM_PRIOR), rt) for rt, model in grid} + self._cells: Final[dict[tuple[RequestType, str], BanditCell]] = cells # mutable-ok: replaced on update + self.pulls: Final[dict[str, int]] = {m: 0 for m in self._models} # mutable-ok: counters + + @property + def models(self) -> tuple[str, ...]: + return self._models + + async def select(self, context: ProgramContext) -> str: + cells: Final = {model: self._cells[(context.request_type, model)] for model in self._models} + chosen: Final = pick_best(cells, self._costs, self._weights.quality, self._weights.cost, rng=self._rng) + self.pulls[chosen] += 1 + return chosen + + def update(self, context: ProgramContext, model: str, score: float) -> None: + _check_model(self._models, model) + key: Final = (context.request_type, model) + self._cells[key] = apply_delta(self._cells[key], score, 1.0 - score) + + def estimate(self, context: ProgramContext, model: str) -> float: + _check_model(self._models, model) + return self._cells[(context.request_type, model)].mean + + def snapshot(self) -> Mapping[str, object]: + cells: Final = { + f"{request_type.value}/{model}": {"mean": round(cell.mean, 4), "samples": cell.total_samples} + for (request_type, model), cell in self._cells.items() + if cell.total_samples > 0 + } + return { + "type": "thompson", + "weights": {"quality": self._weights.quality, "cost": self._weights.cost}, + "pulls": dict(self.pulls), + "cells": cells, + } + + +class FixedDecisionMaker: + """Always the same model; the single-model baseline that still gets verified and reported.""" + + def __init__(self, models: Sequence[str], model: str) -> None: + _check_model(models, model) + self._models: Final[tuple[str, ...]] = tuple(models) + self._model: Final[str] = model + self._updates: int = 0 + + @property + def models(self) -> tuple[str, ...]: + return self._models + + async def select(self, context: ProgramContext) -> str: + return self._model + + def update(self, context: ProgramContext, model: str, score: float) -> None: + _check_model(self._models, model) + self._updates += 1 + + def snapshot(self) -> Mapping[str, object]: + return {"type": "fixed", "model": self._model, "updates": self._updates} + + +class PreRoutingDecisionMaker: + """Let one of LiteLLM's own strategy routers propose the model, once per program. + + The wrapped router (complexity, quality, adaptive or semantic) sees the program's initial request as a + single user message. Its per-request decision becomes a per-program binding, and the program's verified + outcome is recorded against that model, so the state endpoint shows what the strategy actually achieved. + """ + + def __init__(self, models: Sequence[str], router_name: str, strategy: PreRoutingStrategy) -> None: + if not models: + raise ValueError("a decision maker needs at least one model") + self._models: Final[tuple[str, ...]] = tuple(models) + self._router_name: Final[str] = router_name + self._strategy: Final[PreRoutingStrategy] = strategy + self._proposals: Final[dict[str, int]] = {} # mutable-ok: proposal counts per model, incremented per program + self._fallbacks: int = 0 + self._updates: int = 0 + + @property + def models(self) -> tuple[str, ...]: + return self._models + + async def select(self, context: ProgramContext) -> str: + response: Final = await self._strategy.async_pre_routing_hook( + model=self._router_name, + request_kwargs={"metadata": {}}, + messages=[{"role": "user", "content": context.prompt}], + ) + proposed: Final = response.model if response is not None and response.model in self._models else None + if proposed is None: + self._fallbacks += 1 + chosen: Final = proposed if proposed is not None else self._models[0] + self._proposals[chosen] = self._proposals.get(chosen, 0) + 1 + return chosen + + def update(self, context: ProgramContext, model: str, score: float) -> None: + _check_model(self._models, model) + self._updates += 1 + + def snapshot(self) -> Mapping[str, object]: + return { + "type": "pre_routing", + "router": self._router_name, + "proposals": dict(self._proposals), + "fallbacks": self._fallbacks, + "updates": self._updates, + } + + +def load_object(path: str) -> object: + """Import ``package.module:attribute``.""" + module_name, separator, attribute = path.partition(":") + if not separator or not module_name or not attribute: + raise ValueError(f"expected 'package.module:Attribute', got {path!r}") + loaded: Final[object] = getattr(importlib.import_module(module_name), attribute) # pyright: ignore[reportAny] # module attributes are untyped + return loaded + + +def build_decision_maker( + config: OracleDecisionMakerConfig, + models: Sequence[str], + model_costs: Mapping[str, float], + resolve_strategy: Callable[[str], PreRoutingStrategy | None], + model_prefs: Mapping[str, AdaptiveRouterPreferences] | None = None, +) -> DecisionMaker: + """Instantiate the configured decision maker over ``models`` (strongest first).""" + match config.type: + case "thompson": + return ThompsonDecisionMaker( + models, model_costs, weights=config.weights, model_prefs=model_prefs, seed=config.seed + ) + case "fixed": + return FixedDecisionMaker(models, model=config.model or models[0]) + case "pre_routing": + router_name: Final = config.router or "" + strategy: Final = resolve_strategy(router_name) + if strategy is None: + raise ValueError( + f"oracle_router decision_maker.router {router_name!r} is not a configured strategy router" + ) + return PreRoutingDecisionMaker(models, router_name=router_name, strategy=strategy) + case "custom": + factory: Final = load_object(config.path or "") + if not callable(factory): + raise TypeError(f"oracle_router decision_maker.path {config.path!r} is not callable") + instance: Final = factory(tuple(models)) + if not isinstance(instance, DecisionMaker): + raise TypeError(f"{config.path!r} must build an object with select, update, snapshot and models") + return instance diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py new file mode 100644 index 00000000000..a02c664629c --- /dev/null +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -0,0 +1,88 @@ +"""Post-call hook for the ORACLE router. + +After every completion it adds LiteLLM's computed spend and the latest answer to the program's row, and +when the request carried ``metadata.program_done`` it completes the program: the slot is released at once +and the verifier runs in the background. Nothing here can fail a request. +""" + +from collections.abc import Mapping +from typing import TYPE_CHECKING, Final + +from litellm._logging import verbose_router_logger +from litellm.integrations.custom_logger import CustomLogger +from litellm.router_strategy.oracle_router.config import ( + CHOSEN_MODEL_METADATA_KEY, + PROGRAM_DONE_KEY, + PROGRAM_ID_METADATA_KEY, + PROGRAM_SCORE_KEY, + RESPONSE_HEADER, +) +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, as_str_mapping +from litellm.router_strategy.oracle_router.verifier import response_text + +if TYPE_CHECKING: + from litellm.proxy._types import UserAPIKeyAuth + + +def _request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: + return as_str_mapping(as_str_mapping(kwargs.get("litellm_params")).get("metadata")) + + +def _score(metadata: Mapping[str, object]) -> float | None: + raw: Final = metadata.get(PROGRAM_SCORE_KEY) + if isinstance(raw, bool) or not isinstance(raw, (int, float, str)): + return None + try: + return float(raw) + except ValueError: + return None + + +class OracleRouterPostCallHook(CustomLogger): + """One hook per OracleRouter, registered in ``litellm.callbacks``.""" + + def __init__(self, oracle_router: OracleRouter) -> None: + self.oracle_router: Final[OracleRouter] = oracle_router + + async def async_post_call_response_headers_hook( + self, + data: Mapping[str, object], + user_api_key_dict: "UserAPIKeyAuth", + response: object, + request_headers: dict[str, str] | None = None, # mutable-ok: CustomLogger's hook signature + litellm_call_info: dict[str, object] | None = None, # mutable-ok: CustomLogger's hook signature + ) -> dict[str, str] | None: # mutable-ok: CustomLogger's hook signature + buckets: Final = (as_str_mapping(data.get("litellm_metadata")), as_str_mapping(data.get("metadata"))) + chosen: Final = next( + (bucket[CHOSEN_MODEL_METADATA_KEY] for bucket in buckets if CHOSEN_MODEL_METADATA_KEY in bucket), None + ) + return {RESPONSE_HEADER: str(chosen)} if chosen else None + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs, response_obj, succeeded=True) + + async def async_log_failure_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs, response_obj, succeeded=False) + + def _record(self, kwargs: Mapping[str, object], response_obj: object, succeeded: bool) -> None: + try: + metadata: Final = _request_metadata(kwargs) + program_id: Final = metadata.get(PROGRAM_ID_METADATA_KEY) + if not isinstance(program_id, str): + return + if succeeded: + cost: Final = kwargs.get("response_cost") + self.oracle_router.observe_request( + program_id, + cost=float(cost) if isinstance(cost, (int, float)) else 0.0, + response_text=response_text(response_obj), + messages=kwargs.get("messages"), + ) + if metadata.get(PROGRAM_DONE_KEY): + self.oracle_router.complete(program_id, score=_score(metadata)) + except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes + verbose_router_logger.exception("OracleRouterPostCallHook: failed to record request: %s", error) diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py new file mode 100644 index 00000000000..acad4d6599a --- /dev/null +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -0,0 +1,357 @@ +"""The ORACLE router: one binding per agentic program, learned from verified, delayed feedback. + +A program is one agentic task made of many LLM requests that share a prefix. Its first request is bound +to a model by the decision maker and every later request with the same program id reuses that binding. +When the program finishes, the verifier scores it off the critical path, the score and LiteLLM's spend +for the program become one reward, and the decision maker learns from it in arrival order. + +Paper: ORACLE, https://arxiv.org/abs/2607.22465 +""" + +import asyncio +import time +from collections import deque +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, replace +from types import MappingProxyType +from typing import Final + +from pydantic import TypeAdapter, ValidationError + +from litellm._logging import verbose_router_logger +from litellm.litellm_core_utils.core_helpers import ( + get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict +) +from litellm.router_strategy.adaptive_router.classifier import classify_prompt +from litellm.router_strategy.oracle_router.config import ( + CHOSEN_MODEL_METADATA_KEY, + FEEDBACK_HISTORY_SIZE, + PROGRAM_ID_HEADER, + PROGRAM_ID_METADATA_KEY, + PROGRAM_SWEEP_THRESHOLD, + SESSION_ID_FALLBACK_KEYS, +) +from litellm.router_strategy.oracle_router.decision import DecisionMaker, ProgramContext +from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, Verifier, clamp_score +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import OracleRouterConfig, PreRoutingHookResponse, RequestType +from litellm.types.utils import RoutingDecisionCause, StandardLoggingRoutingDecision + +_LEARNING_DECISION_MAKERS: Final[frozenset[str]] = frozenset({"thompson"}) +_MESSAGES: Final = TypeAdapter(Sequence[AllMessageValues]) +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_OBJECTS: Final = TypeAdapter(Sequence[object]) + + +def as_str_mapping(value: object) -> Mapping[str, object]: + """A typed, read-only view of a request-shaped mapping; empty for anything else.""" + if not isinstance(value, Mapping): + return _EMPTY + try: + return _STR_MAPPING.validate_python(value) + except ValidationError: + return _EMPTY + + +@dataclass(frozen=True, slots=True) +class ProgramBinding: + """One row of the lookup table: program id to model, plus what the proxy has seen of the program.""" + + program_id: str + model: str + context: ProgramContext + api_key_hash: str | None + bound_at: float + requests: int = 0 + cost: float = 0.0 + last_response_text: str = "" + last_messages: Sequence[AllMessageValues] = () + + +@dataclass(frozen=True, slots=True) +class FeedbackRecord: + program_id: str + model: str + request_type: str + score: float + cost: float + verified_at: float + + +def first_user_text(messages: Sequence[Mapping[str, object]] | None) -> str: + """The program's initial request: the first user message, text parts only.""" + for message in messages or (): + if message.get("role") != "user": + continue + content = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, Sequence): + parts = tuple(as_str_mapping(part) for part in _OBJECTS.validate_python(content)) + return "\n".join(str(part.get("text", "")) for part in parts if part.get("type") == "text") + return "" + return "" + + +def _metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + """The request's metadata as callers see it, with the proxy-internal bucket layered on top.""" + return {**as_str_mapping(request_kwargs.get("metadata")), **as_str_mapping(request_kwargs.get("litellm_metadata"))} + + +def _header(request_kwargs: Mapping[str, object], name: str) -> str | None: + headers: Final = as_str_mapping(request_kwargs.get("headers")) + matches: Final = tuple(value for key, value in headers.items() if key.lower() == name) + return str(matches[0]) if matches else None + + +def resolve_program_id(request_kwargs: Mapping[str, object], program_id_key: str) -> str | None: + """The program id of a request: ``metadata[program_id_key]``, then the session id spellings, then the header.""" + metadata: Final = _metadata(request_kwargs) + candidates: Final = ( + metadata.get(program_id_key), + *(metadata.get(key) for key in SESSION_ID_FALLBACK_KEYS), + request_kwargs.get("litellm_session_id"), + _header(request_kwargs, PROGRAM_ID_HEADER), + ) + return next((str(candidate) for candidate in candidates if candidate), None) + + +def _transcript(messages: object, fallback: Sequence[AllMessageValues]) -> Sequence[AllMessageValues]: + if not messages: + return fallback + try: + return tuple(_MESSAGES.validate_python(messages)) + except ValidationError: + return fallback + + +def _api_key_hash(request_kwargs: Mapping[str, object]) -> str | None: + value: Final = _metadata(request_kwargs).get("user_api_key_hash") + return str(value) if value else None + + +class OracleRouter: + """One instance per ``auto_router/oracle_router`` deployment.""" + + def __init__( + self, + router_name: str, + config: OracleRouterConfig, + decision_maker: DecisionMaker, + verifier: Verifier, + clock: Callable[[], float] = time.time, + ) -> None: + self.router_name: Final[str] = router_name + self.config: Final[OracleRouterConfig] = config + self.decision_maker: Final[DecisionMaker] = decision_maker + self.verifier: Final[Verifier] = verifier + self._clock: Final = clock + self._cause: Final[RoutingDecisionCause] = ( + "bandit" if config.decision_maker.type in _LEARNING_DECISION_MAKERS else "classifier_plugin" + ) + self._bindings: Final[dict[str, ProgramBinding]] = {} # mutable-ok: the lookup table + self._tasks: Final[set[asyncio.Task[float]]] = set() # mutable-ok: in-flight verifications + self._history: Final[deque[FeedbackRecord]] = deque(maxlen=FEEDBACK_HISTORY_SIZE) # mutable-ok: bounded log + self._update_lock: Final = asyncio.Lock() + self.programs_bound: int = 0 + self.programs_completed: int = 0 + self.programs_evicted: int = 0 + self.stateless_requests: int = 0 + self.verifications_failed: int = 0 + + @property + def models(self) -> tuple[str, ...]: + return tuple(self.config.available_models) + + def context_for(self, program_id: str, prompt: str) -> ProgramContext: + return ProgramContext(program_id=program_id, prompt=prompt, request_type=classify_prompt(prompt)) + + def binding(self, program_id: str) -> ProgramBinding | None: + return self._bindings.get(program_id) + + def _evict_expired(self, now: float) -> None: + deadline: Final = now - self.config.program_ttl_seconds + expired: Final = tuple( + program_id for program_id, binding in self._bindings.items() if binding.bound_at < deadline + ) + for program_id in expired: + self._bindings.pop(program_id, None) + overflow: Final = len(self._bindings) - self.config.max_programs + oldest: Final = tuple(sorted(self._bindings, key=lambda pid: self._bindings[pid].bound_at))[: max(overflow, 0)] + for program_id in oldest: + self._bindings.pop(program_id, None) + self.programs_evicted += len(expired) + len(oldest) + + async def bind(self, program_id: str, prompt: str, api_key_hash: str | None) -> ProgramBinding: + """Bind a new program, or return its existing row.""" + existing: Final = self._bindings.get(program_id) + if existing is not None: + return existing + now: Final = self._clock() + if len(self._bindings) >= min(PROGRAM_SWEEP_THRESHOLD, self.config.max_programs): + self._evict_expired(now) + context: Final = self.context_for(program_id, prompt) + model: Final = await self.decision_maker.select(context) + binding: Final = ProgramBinding( + program_id=program_id, model=model, context=context, api_key_hash=api_key_hash, bound_at=now + ) + self._bindings[program_id] = binding + self.programs_bound += 1 + return binding + + async def async_pre_routing_hook( + self, + model: str, + request_kwargs: dict[str, object], # mutable-ok: pre-routing contract + messages: list[dict[str, object]] | None = None, # mutable-ok: pre-routing contract + input: str | list[object] | None = None, # mutable-ok: pre-routing contract + specific_deployment: bool | None = False, + ) -> PreRoutingHookResponse | None: + """Route the request to its program's model, binding the program on its first request.""" + program_id: Final = resolve_program_id(request_kwargs, self.config.program_id_key) + prompt: Final = first_user_text(messages) + chosen_model, request_type = await self._decide(program_id, prompt, _api_key_hash(request_kwargs)) + _, bucket = get_or_create_metadata_bucket(request_kwargs) # pyright: ignore[reportUnknownVariableType] # bare dict upstream + bucket[CHOSEN_MODEL_METADATA_KEY] = chosen_model + bucket[PROGRAM_ID_METADATA_KEY] = program_id + verbose_router_logger.debug( + "OracleRouter[%s]: program=%s request_type=%s -> %s", + self.router_name, + program_id, + request_type.value, + chosen_model, + ) + return PreRoutingHookResponse( + model=chosen_model, + messages=messages, + routing_decision=StandardLoggingRoutingDecision( + router_model_name=self.router_name, + router_type="oracle", + routed_model=chosen_model, + cause=self._cause, + request_type=request_type.value, + ), + ) + + async def _decide(self, program_id: str | None, prompt: str, api_key_hash: str | None) -> tuple[str, RequestType]: + if program_id is None: + self.stateless_requests += 1 + context: Final = self.context_for("", prompt) + return await self.decision_maker.select(context), context.request_type + binding: Final = await self.bind(program_id, prompt, api_key_hash) + return binding.model, binding.context.request_type + + def observe_request(self, program_id: str, cost: float, response_text: str, messages: object = None) -> None: + """Account one finished LLM request of the program: LiteLLM's spend, the latest answer and transcript.""" + binding: Final = self._bindings.get(program_id) + if binding is None: + return + transcript: Final = _transcript(messages, binding.last_messages) + self._bindings[program_id] = replace( + binding, + requests=binding.requests + 1, + cost=binding.cost + max(cost, 0.0), + last_response_text=response_text or binding.last_response_text, + last_messages=transcript, + ) + + def complete( + self, + program_id: str, + score: float | None = None, + cost: float | None = None, + payload: Mapping[str, object] = _EMPTY, + ) -> asyncio.Task[float] | None: + """Release the program now and verify it in the background; the task resolves to the verified score.""" + binding: Final = self._bindings.pop(program_id, None) + if binding is None: + return None + outcome: Final = ProgramOutcome( + program_id=program_id, + model=binding.model, + prompt=binding.context.prompt, + response_text=binding.last_response_text, + cost=cost if cost is not None else binding.cost, + messages=binding.last_messages, + score=score, + payload=payload, + ) + self.programs_completed += 1 + task: Final = asyncio.create_task(self._verify_and_learn(binding, outcome)) + self._tasks.add(task) + task.add_done_callback(self._tasks.discard) + return task + + async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float: + try: + score: Final = clamp_score(await self.verifier.verify(outcome)) + except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop + self.verifications_failed += 1 + verbose_router_logger.exception( + "OracleRouter[%s]: verification of %s failed: %s", self.router_name, binding.program_id, error + ) + return float("nan") + async with self._update_lock: + self.decision_maker.update(binding.context, binding.model, score) + self._history.append( + FeedbackRecord( + program_id=binding.program_id, + model=binding.model, + request_type=binding.context.request_type.value, + score=score, + cost=outcome.cost, + verified_at=self._clock(), + ) + ) + return score + + async def drain(self) -> None: + """Wait for every in-flight verification.""" + while self._tasks: + await asyncio.gather(*tuple(self._tasks), return_exceptions=True) + + @property + def pending_verifications(self) -> int: + return len(self._tasks) + + def _accuracy_by_model(self) -> Mapping[str, float]: + scores: Final[dict[str, list[float]]] = {} # mutable-ok: grouping accumulator for the snapshot + for record in self._history: + scores.setdefault(record.model, []).append(record.score) + return {model: round(sum(values) / len(values), 4) for model, values in scores.items()} + + async def get_state_snapshot(self) -> Mapping[str, object]: + """In-memory view for the introspection endpoint.""" + active: Final[dict[str, int]] = {} # mutable-ok: counting accumulator for the snapshot + for binding in self._bindings.values(): + active[binding.model] = active.get(binding.model, 0) + 1 + return { + "router_name": self.router_name, + "available_models": list(self.models), + "decision_maker": self.decision_maker.snapshot(), + "verifier": type(self.verifier).__name__, + "active_programs": len(self._bindings), + "active_per_model": active, + "programs_bound": self.programs_bound, + "programs_completed": self.programs_completed, + "programs_evicted": self.programs_evicted, + "stateless_requests": self.stateless_requests, + "pending_verifications": self.pending_verifications, + "verifications_failed": self.verifications_failed, + "recent_accuracy": self._accuracy_by_model(), + "recent_feedback": [ + { + "program_id": record.program_id, + "model": record.model, + "request_type": record.request_type, + "score": record.score, + "cost": record.cost, + } + for record in tuple(self._history)[-20:] + ], + } + + +def request_type_of(prompt: str) -> RequestType: + return classify_prompt(prompt) diff --git a/litellm/router_strategy/oracle_router/verifier.py b/litellm/router_strategy/oracle_router/verifier.py new file mode 100644 index 00000000000..f35a30c0ec9 --- /dev/null +++ b/litellm/router_strategy/oracle_router/verifier.py @@ -0,0 +1,120 @@ +"""Verifiers score a finished program in [0, 1], off the critical path. + +The contract is one coroutine. ORACLE ships three: any configured LiteLLM guardrail applied to the +program's final turn with its transcript as context (``llm_as_a_judge``, ``custom_code`` or a hosted +check), the score the client reported, and an import path for an operator's own class. +""" + +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Final, Protocol, runtime_checkable + +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import OracleVerifierConfig +from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse + +from .decision import load_object + +_EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class ProgramOutcome: + """Everything a verifier may look at once a program has finished.""" + + program_id: str + model: str + prompt: str + response_text: str + cost: float + messages: Sequence[AllMessageValues] = () + score: float | None = None + payload: Mapping[str, object] = field(default_factory=lambda: _EMPTY) + + +@runtime_checkable +class Verifier(Protocol): + async def verify(self, outcome: ProgramOutcome) -> float: ... + + +def clamp_score(value: float) -> float: + return min(max(float(value), 0.0), 1.0) + + +def response_text(response: object) -> str: + """The assistant text of a chat completion, or an empty string for anything else.""" + if not isinstance(response, ModelResponse) or not response.choices: + return "" + content: Final = response.choices[0].message.content + return content if isinstance(content, str) else "" + + +class ReportedVerifier: + """Trust the score the client reported with the program's last request or through the feedback endpoint.""" + + def __init__(self, default: float = 0.0) -> None: + self._default: Final[float] = default + + async def verify(self, outcome: ProgramOutcome) -> float: + return clamp_score(outcome.score) if outcome.score is not None else self._default + + +def find_guardrail(name: str) -> CustomGuardrail | None: + """The initialized guardrail registered under ``name``, if any.""" + callbacks: Final = litellm.logging_callback_manager.get_custom_loggers_for_type(callback_type=CustomGuardrail) + guardrails: Final = tuple(callback for callback in callbacks if isinstance(callback, CustomGuardrail)) + return next((guardrail for guardrail in guardrails if guardrail.guardrail_name == name), None) + + +class GuardrailVerifier: + """Apply a configured LiteLLM guardrail to the program's final turn: it passes or it is blocked. + + The guardrail sees the final assistant text as ``texts``, the program's transcript as + ``structured_messages`` and the feedback payload as ``request_data["metadata"]``, so an + ``llm_as_a_judge`` guardrail judges the whole conversation and a ``custom_code`` guardrail can read + whatever the harness posted with the feedback. + """ + + def __init__(self, guardrail_name: str, lookup: Callable[[str], CustomGuardrail | None] = find_guardrail) -> None: + self._name: Final[str] = guardrail_name + self._lookup: Final = lookup + + async def verify(self, outcome: ProgramOutcome) -> float: + guardrail: Final = self._lookup(self._name) + if guardrail is None: + raise LookupError(f"guardrail {self._name!r} is not initialized on this proxy") + transcript: Final = list(outcome.messages) or [{"role": "user", "content": outcome.prompt}] + inputs: Final = GenericGuardrailAPIInputs( + texts=[outcome.response_text], model=outcome.model, structured_messages=transcript + ) + try: + await guardrail.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # upstream types request_data as a bare dict + inputs=inputs, + request_data={ + "model": outcome.model, + "metadata": {"program_id": outcome.program_id, **outcome.payload}, + }, + input_type="response", + ) + except Exception: # noqa: BLE001 # a guardrail signals "blocked" by raising its own exception type + return 0.0 + return 1.0 + + +def build_verifier(config: OracleVerifierConfig) -> Verifier: + match config.type: + case "reported": + return ReportedVerifier(default=config.default_score) + case "guardrail": + return GuardrailVerifier(config.guardrail or "") + case "custom": + factory: Final = load_object(config.path or "") + if not callable(factory): + raise TypeError(f"oracle_router verifier.path {config.path!r} is not callable") + instance: Final = factory() + if not isinstance(instance, Verifier): + raise TypeError(f"{config.path!r} must build an object with an async verify(outcome) method") + return instance diff --git a/litellm/router_utils/auto_router_model_naming.py b/litellm/router_utils/auto_router_model_naming.py index 3423836e4fd..028252bed00 100644 --- a/litellm/router_utils/auto_router_model_naming.py +++ b/litellm/router_utils/auto_router_model_naming.py @@ -24,7 +24,7 @@ from litellm.router_strategy.complexity_router.config import ( AUTO_ROUTER_MODEL_PREFIX: Final = "auto_router/" -StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality"] +StrategyRouterKind = Literal["semantic", "complexity", "adaptive", "quality", "oracle"] StrategyRouterDependencyRole: TypeAlias = Literal[ "tier", "default", "classifier", "embedding", "evaluation", "compactor" @@ -51,6 +51,7 @@ STRATEGY_ROUTER_PARAM_FIELDS: Final[frozenset[str]] = frozenset( "adaptive_router_config", "quality_router_config", "quality_router_default_model", + "oracle_router_config", } ) @@ -63,6 +64,7 @@ _REQUIRED_FIELD_GROUPS: Final[Mapping[StrategyRouterKind, tuple[tuple[str, ...], "complexity": (("complexity_router_config", "complexity_router_default_model"),), "adaptive": (("adaptive_router_config",),), "quality": (("quality_router_config", "quality_router_default_model"),), + "oracle": (("oracle_router_config",),), } @@ -82,6 +84,8 @@ def classify_strategy_router_model(model: str) -> StrategyRouterKind | None: return "adaptive" if remainder.startswith("quality_router"): return "quality" + if remainder.startswith("oracle_router"): + return "oracle" return "semantic" @@ -141,6 +145,8 @@ def strategy_router_dependencies( ) if kind == "adaptive": return _pool(_mapping(litellm_params.get("adaptive_router_config")).get("available_models"), "tier") + if kind == "oracle": + return _pool(_mapping(litellm_params.get("oracle_router_config")).get("available_models"), "tier") if kind == "quality": quality: Final = _mapping(litellm_params.get("quality_router_config")) return tuple( diff --git a/litellm/types/management_endpoints/oracle_router_endpoints.py b/litellm/types/management_endpoints/oracle_router_endpoints.py new file mode 100644 index 00000000000..cb5f63faf35 --- /dev/null +++ b/litellm/types/management_endpoints/oracle_router_endpoints.py @@ -0,0 +1,27 @@ +"""Types for the ORACLE router management endpoints.""" + +from collections.abc import Mapping, Sequence + +from pydantic import BaseModel, Field + + +class OracleRouterFeedbackRequest(BaseModel): + """Report that a program finished. ``score`` is optional when the router's verifier grades it.""" + + program_id: str = Field(min_length=1) + score: float | None = Field(default=None, ge=0.0, le=1.0) + cost: float | None = Field( + default=None, ge=0.0, description="Overrides the spend LiteLLM accumulated for the program" + ) + payload: Mapping[str, object] = Field(default_factory=dict, description="Passed to the verifier unchanged") + + +class OracleRouterFeedbackResponse(BaseModel): + program_id: str + router_name: str + model: str + pending_verifications: int + + +class OracleRouterStateResponse(BaseModel): + routers: Sequence[Mapping[str, object]] diff --git a/litellm/types/router.py b/litellm/types/router.py index 2ab1a1185ed..e4dee4e0085 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -6,7 +6,7 @@ import datetime import enum from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, Generic, Literal, TypeVar, get_type_hints +from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Final, Generic, Literal, TypeAlias, TypeVar, get_type_hints from zoneinfo import ZoneInfo, ZoneInfoNotFoundError import httpx @@ -451,6 +451,8 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): # adaptive-router params adaptive_router_default_model: str | None = None adaptive_router_config: dict | None = None + # oracle-router params + oracle_router_config: Mapping[str, object] | None = None # quality-router params quality_router_config: dict | None = None quality_router_default_model: str | None = None @@ -1243,3 +1245,75 @@ class AdaptiveRouterPreferences(BaseModel): quality_tier: int = Field(ge=1, le=3) strengths: list[RequestType] = Field(default_factory=list) + + +OracleDecisionMakerType: TypeAlias = Literal["thompson", "fixed", "pre_routing", "custom"] +OracleVerifierType: TypeAlias = Literal["reported", "guardrail", "custom"] + + +class OracleDecisionMakerConfig(BaseModel): + """Which policy binds a program to a model, and its knobs. + + ``thompson`` (default) is the adaptive router's bandit trained on verified program outcomes; ``weights`` + are that bandit's quality/cost weights. ``fixed`` always picks ``model``. ``pre_routing`` lets the strategy + router named by ``router`` (a complexity, quality, adaptive or semantic router in the same model_list) + propose the model once per program. ``custom`` imports ``path`` (``package.module:Factory``) and calls it + with the model tuple. + """ + + type: OracleDecisionMakerType = "thompson" + weights: AdaptiveRouterWeights = Field(default_factory=AdaptiveRouterWeights) + seed: int = 0 + model: str | None = None + router: str | None = None + path: str | None = None + + @model_validator(mode="after") + def _required_fields(self) -> "OracleDecisionMakerConfig": + required: Final = {"fixed": self.model, "pre_routing": self.router, "custom": self.path} + if self.type in required and not required[self.type]: + names: Final = {"fixed": "model", "pre_routing": "router", "custom": "path"} + raise ValueError(f"decision_maker.type={self.type!r} requires decision_maker.{names[self.type]}") + return self + + +class OracleVerifierConfig(BaseModel): + """How a finished program is scored. + + ``guardrail`` applies the configured LiteLLM guardrail ``guardrail`` (for example an ``llm_as_a_judge`` + guardrail) to the program's final turn with the transcript as context. ``reported`` trusts + ``metadata.program_score`` or the feedback endpoint. ``custom`` imports ``path`` and calls it with no + arguments. + """ + + type: OracleVerifierType = "reported" + default_score: float = Field(default=0.0, ge=0.0, le=1.0) + guardrail: str | None = None + path: str | None = None + + @model_validator(mode="after") + def _required_fields(self) -> "OracleVerifierConfig": + required: Final = {"guardrail": self.guardrail, "custom": self.path} + if self.type in required and not required[self.type]: + names: Final = {"guardrail": "guardrail", "custom": "path"} + raise ValueError(f"verifier.type={self.type!r} requires verifier.{names[self.type]}") + return self + + +class OracleRouterConfig(BaseModel): + """``litellm_params.oracle_router_config`` of an ``auto_router/oracle_router`` deployment.""" + + available_models: tuple[str, ...] = Field(min_length=1) + decision_maker: OracleDecisionMakerConfig = Field(default_factory=OracleDecisionMakerConfig) + verifier: OracleVerifierConfig = Field(default_factory=OracleVerifierConfig) + program_id_key: str = "program_id" + program_ttl_seconds: int = Field(default=24 * 3600, ge=1) + max_programs: int = Field(default=10_000, ge=1) + + @model_validator(mode="after") + def _fixed_model_is_available(self) -> "OracleRouterConfig": + if len(self.available_models) != len(frozenset(self.available_models)): + raise ValueError("available_models must be distinct") + if self.decision_maker.type == "fixed" and self.decision_maker.model not in self.available_models: + raise ValueError("decision_maker.model must be one of available_models") + return self diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 8c10b9e3497..f29dc3b059f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3079,7 +3079,7 @@ class StandardLoggingRoutingDecision(TypedDict, total=False): """Per-request provenance for a pre-routing strategy (auto-router) decision.""" router_model_name: str - router_type: Literal["complexity", "adaptive", "quality"] + router_type: Literal["complexity", "adaptive", "quality", "oracle"] routed_model: str cause: RoutingDecisionCause tier: str diff --git a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py new file mode 100644 index 00000000000..a3d73861634 --- /dev/null +++ b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py @@ -0,0 +1,124 @@ +"""POST /oracle_router/feedback and GET /oracle_router/state.""" + +from unittest.mock import patch + +import pytest +from fastapi import HTTPException +from pydantic import ValidationError + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.management_endpoints.oracle_router_endpoints import ( + get_oracle_router_state, + submit_oracle_router_feedback, +) +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter +from litellm.router_strategy.oracle_router.verifier import ReportedVerifier +from litellm.types.management_endpoints.oracle_router_endpoints import OracleRouterFeedbackRequest +from litellm.types.router import OracleRouterConfig, TaggedPreRoutingStrategy + +ADMIN = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, api_key="admin-hash") +OWNER = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="owner-hash") +STRANGER = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, api_key="other-hash") + + +class _Maker: + def __init__(self) -> None: + self.updates = [] + + @property + def models(self): + return ("smart", "fast") + + async def select(self, context): + return "smart" + + def update(self, context, model, score): + self.updates.append((model, score)) + + def snapshot(self): + return {"type": "test"} + + +class _LLMRouter: + def __init__(self, oracle: OracleRouter) -> None: + self.oracle_routers = {oracle.router_name: [TaggedPreRoutingStrategy(tags=(), strategy=oracle)]} + + +@pytest.fixture +def oracle(): + maker = _Maker() + router = OracleRouter("oracle", OracleRouterConfig(available_models=["smart", "fast"]), maker, ReportedVerifier()) + router.maker = maker + return router + + +async def _bind(oracle: OracleRouter, program_id: str, api_key_hash: str | None = "owner-hash"): + metadata = {"program_id": program_id, **({"user_api_key_hash": api_key_hash} if api_key_hash else {})} + await oracle.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": metadata}, messages=[{"role": "user", "content": "t"}] + ) + + +@pytest.mark.asyncio +async def test_feedback_completes_the_program_and_the_router_learns(oracle): + await _bind(oracle, "task-1") + oracle.observe_request("task-1", cost=0.2, response_text="done") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER + ) + assert response.model_dump() == { + "program_id": "task-1", + "router_name": "oracle", + "model": "smart", + "pending_verifications": 1, + } + assert oracle.binding("task-1") is None + await oracle.drain() + assert oracle.maker.updates == [("smart", 1.0)] + + +@pytest.mark.asyncio +async def test_feedback_from_another_key_is_forbidden_but_an_admin_may_complete(oracle): + await _bind(oracle, "task-1") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + with pytest.raises(HTTPException) as denied: + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), STRANGER) + assert denied.value.status_code == 403 and oracle.binding("task-1") is not None + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=0.0), ADMIN + ) + assert response.program_id == "task-1" and oracle.binding("task-1") is None + await oracle.drain() + + +@pytest.mark.asyncio +async def test_feedback_for_an_unknown_program_or_without_routers_is_404(oracle): + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + with pytest.raises(HTTPException) as unknown: + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="nope"), OWNER) + assert unknown.value.status_code == 404 + with patch("litellm.proxy.proxy_server.llm_router", None): + with pytest.raises(HTTPException) as none: + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="nope"), OWNER) + assert none.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_state_is_admin_only_and_lists_every_router(oracle): + await _bind(oracle, "task-1") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + with pytest.raises(HTTPException) as denied: + await get_oracle_router_state(OWNER) + assert denied.value.status_code == 403 + state = await get_oracle_router_state(ADMIN) + assert len(state.routers) == 1 + assert state.routers[0]["router_name"] == "oracle" and state.routers[0]["active_programs"] == 1 + assert state.routers[0]["decision_maker"] == {"type": "test"} + + +def test_feedback_request_validates_score_range(): + with pytest.raises(ValidationError, match="score"): + OracleRouterFeedbackRequest(program_id="p", score=1.5) + with pytest.raises(ValidationError, match="program_id"): + OracleRouterFeedbackRequest(program_id="") diff --git a/tests/unit/router_strategy/oracle_router/__init__.py b/tests/unit/router_strategy/oracle_router/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/router_strategy/oracle_router/test_decision.py b/tests/unit/router_strategy/oracle_router/test_decision.py new file mode 100644 index 00000000000..5e17d3fc9c8 --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_decision.py @@ -0,0 +1,171 @@ +"""Decision makers: each learns or proposes the way its contract says.""" + +import pytest + +from litellm.router_strategy.oracle_router.decision import ( + DecisionMaker, + FixedDecisionMaker, + PreRoutingDecisionMaker, + ProgramContext, + ThompsonDecisionMaker, + build_decision_maker, + load_object, +) +from litellm.types.router import AdaptiveRouterWeights, OracleDecisionMakerConfig, PreRoutingHookResponse, RequestType + +MODELS = ("smart", "fast") +COSTS = {"smart": 1e-5, "fast": 1e-6} + + +def _ctx(prompt: str = "fix the failing test", request_type: RequestType = RequestType.GENERAL) -> ProgramContext: + return ProgramContext(program_id="p", prompt=prompt, request_type=request_type) + + +@pytest.mark.asyncio +async def test_thompson_learns_a_different_winner_per_request_type(): + maker = ThompsonDecisionMaker(MODELS, COSTS, weights=AdaptiveRouterWeights(quality=1.0, cost=0.0), seed=0) + for _ in range(80): + for request_type, good in ((RequestType.CODE_GENERATION, "smart"), (RequestType.WRITING, "fast")): + ctx = _ctx(request_type=request_type) + chosen = await maker.select(ctx) + maker.update(ctx, chosen, 1.0 if chosen == good else 0.0) + code, writing = _ctx(request_type=RequestType.CODE_GENERATION), _ctx(request_type=RequestType.WRITING) + assert maker.estimate(code, "smart") > maker.estimate(code, "fast") + assert maker.estimate(writing, "fast") > maker.estimate(writing, "smart") + picks = [await maker.select(code) for _ in range(30)] + assert picks.count("smart") > 24 + assert maker.snapshot()["cells"]["code_generation/smart"]["samples"] > 0 + + +@pytest.mark.asyncio +async def test_thompson_cost_weight_prefers_the_cheap_model_when_quality_ties(): + maker = ThompsonDecisionMaker(MODELS, COSTS, weights=AdaptiveRouterWeights(quality=0.3, cost=0.7), seed=1) + ctx = _ctx() + for _ in range(40): + maker.update(ctx, "smart", 0.8) + maker.update(ctx, "fast", 0.8) + picks = [await maker.select(ctx) for _ in range(30)] + assert picks.count("fast") > 24 + + +@pytest.mark.asyncio +async def test_thompson_priors_come_from_adaptive_router_preferences(): + from litellm.types.router import AdaptiveRouterPreferences + + prefs = { + "smart": AdaptiveRouterPreferences(quality_tier=3, strengths=[]), + "fast": AdaptiveRouterPreferences(quality_tier=1, strengths=[]), + } + maker = ThompsonDecisionMaker( + MODELS, COSTS, weights=AdaptiveRouterWeights(quality=1.0, cost=0.0), model_prefs=prefs, seed=0 + ) + ctx = _ctx() + assert maker.estimate(ctx, "smart") > maker.estimate(ctx, "fast") + picks = [await maker.select(ctx) for _ in range(30)] + assert picks.count("smart") > 24 + + +def test_thompson_rejects_unknown_model_and_empty_pool(): + maker = ThompsonDecisionMaker(MODELS, COSTS) + with pytest.raises(KeyError): + maker.update(_ctx(), "other", 1.0) + with pytest.raises(ValueError, match="at least one model"): + ThompsonDecisionMaker((), COSTS) + + +@pytest.mark.asyncio +async def test_fixed_always_returns_its_model_and_counts_updates(): + maker = FixedDecisionMaker(MODELS, model="fast") + assert await maker.select(_ctx()) == "fast" + maker.update(_ctx(), "fast", 1.0) + assert maker.snapshot() == {"type": "fixed", "model": "fast", "updates": 1} + with pytest.raises(KeyError): + FixedDecisionMaker(MODELS, model="nope") + + +class _FakeStrategy: + def __init__(self, answer: str | None) -> None: + self.answer = answer + self.calls: list[dict] = [] + + async def async_pre_routing_hook(self, model, request_kwargs, messages=None, input=None, specific_deployment=False): + self.calls.append({"model": model, "messages": messages}) + return None if self.answer is None else PreRoutingHookResponse(model=self.answer, messages=messages) + + +@pytest.mark.asyncio +async def test_pre_routing_uses_the_wrapped_strategy_proposal_once_per_program(): + strategy = _FakeStrategy("fast") + maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy) + assert await maker.select(_ctx("write a haiku")) == "fast" + assert strategy.calls == [{"model": "tiered", "messages": [{"role": "user", "content": "write a haiku"}]}] + assert maker.snapshot()["proposals"] == {"fast": 1} + + +@pytest.mark.asyncio +async def test_pre_routing_falls_back_to_the_first_model_when_the_proposal_is_unusable(): + assert await PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy(None)).select(_ctx()) == "smart" + maker = PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy("not-configured")) + assert await maker.select(_ctx()) == "smart" + assert maker.snapshot()["fallbacks"] == 1 + + +class CustomMaker: + def __init__(self, models: tuple[str, ...]) -> None: + self._models = models + + @property + def models(self) -> tuple[str, ...]: + return self._models + + async def select(self, context: ProgramContext) -> str: + return self._models[-1] + + def update(self, context: ProgramContext, model: str, score: float) -> None: + pass + + def snapshot(self): + return {"type": "custom"} + + +class NotAMaker: + def __init__(self, models: tuple[str, ...]) -> None: + pass + + +@pytest.mark.asyncio +async def test_build_decision_maker_by_type(): + resolve = {"tiered": _FakeStrategy("smart")}.get + thompson = build_decision_maker(OracleDecisionMakerConfig(), MODELS, COSTS, resolve) + assert isinstance(thompson, ThompsonDecisionMaker) and await thompson.select(_ctx()) in MODELS + assert isinstance( + build_decision_maker(OracleDecisionMakerConfig(type="fixed", model="fast"), MODELS, COSTS, resolve), + FixedDecisionMaker, + ) + pre = build_decision_maker(OracleDecisionMakerConfig(type="pre_routing", router="tiered"), MODELS, COSTS, resolve) + assert isinstance(pre, PreRoutingDecisionMaker) and await pre.select(_ctx()) == "smart" + with pytest.raises(ValueError, match="not a configured strategy router"): + build_decision_maker(OracleDecisionMakerConfig(type="pre_routing", router="missing"), MODELS, COSTS, resolve) + custom = build_decision_maker( + OracleDecisionMakerConfig(type="custom", path=f"{__name__}:CustomMaker"), MODELS, COSTS, resolve + ) + assert isinstance(custom, DecisionMaker) and await custom.select(_ctx()) == "fast" + with pytest.raises(TypeError): + build_decision_maker( + OracleDecisionMakerConfig(type="custom", path=f"{__name__}:NotAMaker"), MODELS, COSTS, resolve + ) + + +def test_load_object_requires_module_colon_attribute(): + with pytest.raises(ValueError, match=r"package\.module:Attribute"): + load_object("no_colon_here") + assert load_object(f"{__name__}:CustomMaker") is CustomMaker + + +def test_config_requires_the_fields_each_type_reads(): + with pytest.raises(ValueError, match=r"decision_maker\.model"): + OracleDecisionMakerConfig(type="fixed") + with pytest.raises(ValueError, match=r"decision_maker\.router"): + OracleDecisionMakerConfig(type="pre_routing") + with pytest.raises(ValueError, match=r"decision_maker\.path"): + OracleDecisionMakerConfig(type="custom") diff --git a/tests/unit/router_strategy/oracle_router/test_hooks.py b/tests/unit/router_strategy/oracle_router/test_hooks.py new file mode 100644 index 00000000000..08360e35f00 --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_hooks.py @@ -0,0 +1,119 @@ +"""The post-call hook feeds spend and completion signals from LiteLLM's logging payload into the router.""" + +import asyncio + +import pytest + +from litellm.router_strategy.oracle_router.config import ( + CHOSEN_MODEL_METADATA_KEY, + PROGRAM_ID_METADATA_KEY, + RESPONSE_HEADER, +) +from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter +from litellm.router_strategy.oracle_router.verifier import ReportedVerifier +from litellm.types.router import OracleRouterConfig +from litellm.types.utils import Choices, Message, ModelResponse + + +class _Maker: + def __init__(self) -> None: + self.updates = [] + + @property + def models(self): + return ("smart", "fast") + + async def select(self, context): + return "smart" + + def update(self, context, model, score): + self.updates.append((model, score)) + + def snapshot(self): + return {} + + +def _kwargs(program_id="t1", cost=0.01, **metadata): + return { + "model": "openai/gpt-4o", + "messages": [{"role": "user", "content": "task"}], + "litellm_params": {"metadata": {PROGRAM_ID_METADATA_KEY: program_id, **metadata}}, + "response_cost": cost, + } + + +def _response(text="answer") -> ModelResponse: + return ModelResponse(choices=[Choices(message=Message(content=text, role="assistant"))]) + + +@pytest.fixture +def bound(): + maker = _Maker() + router = OracleRouter("oracle", OracleRouterConfig(available_models=["smart", "fast"]), maker, ReportedVerifier()) + asyncio.run( + router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "t1"}}, + messages=[{"role": "user", "content": "task"}], + ) + ) + return router, maker, OracleRouterPostCallHook(router) + + +@pytest.mark.asyncio +async def test_success_events_accumulate_spend_and_the_latest_answer(bound): + router, _, hook = bound + await hook.async_log_success_event(_kwargs(cost=0.01), _response("first"), None, None) + await hook.async_log_success_event(_kwargs(cost=0.02), _response("second"), None, None) + binding = router.binding("t1") + assert binding.requests == 2 and binding.cost == pytest.approx(0.03) and binding.last_response_text == "second" + assert list(binding.last_messages) == [{"role": "user", "content": "task"}] + + +@pytest.mark.asyncio +async def test_program_done_completes_the_program_with_the_reported_score(bound): + router, maker, hook = bound + await hook.async_log_success_event( + _kwargs(cost=0.05, program_done=True, program_score="1"), _response(), None, None + ) + assert router.binding("t1") is None + await router.drain() + assert maker.updates == [("smart", 1.0)] + + +@pytest.mark.asyncio +async def test_failure_event_does_not_add_spend_but_still_honours_program_done(bound): + router, maker, hook = bound + await hook.async_log_failure_event(_kwargs(cost=0.5), None, None, None) + assert router.binding("t1").cost == 0.0 + await hook.async_log_failure_event(_kwargs(program_done=True), None, None, None) + await router.drain() + assert maker.updates == [("smart", 0.0)] + + +@pytest.mark.asyncio +async def test_events_for_unknown_or_untagged_programs_are_ignored(bound): + router, _, hook = bound + await hook.async_log_success_event( + {"litellm_params": {"metadata": {}}, "response_cost": 1.0}, _response(), None, None + ) + await hook.async_log_success_event(_kwargs(program_id="other"), _response(), None, None) + assert router.binding("t1").requests == 0 and router.binding("other") is None + + +@pytest.mark.asyncio +async def test_response_header_names_the_bound_model(bound): + _, _, hook = bound + headers = await hook.async_post_call_response_headers_hook( + {"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart"}}, None, None + ) + assert headers == {RESPONSE_HEADER: "smart"} + assert await hook.async_post_call_response_headers_hook({"metadata": {}}, None, None) is None + + +@pytest.mark.asyncio +async def test_response_header_reads_the_litellm_metadata_bucket_too(bound): + _, _, hook = bound + data = {"metadata": {}, "litellm_metadata": {CHOSEN_MODEL_METADATA_KEY: "fast"}} + assert await hook.async_post_call_response_headers_hook(data, None, None) == {RESPONSE_HEADER: "fast"} diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py new file mode 100644 index 00000000000..4d2ab51a2c8 --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -0,0 +1,206 @@ +"""OracleRouter: sticky program bindings, program id resolution, delayed feedback, eviction.""" + +import asyncio + +import pytest + +from litellm.router_strategy.oracle_router.config import ( + CHOSEN_MODEL_METADATA_KEY, + PROGRAM_ID_HEADER, + PROGRAM_ID_METADATA_KEY, +) +from litellm.router_strategy.oracle_router.decision import ProgramContext +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, first_user_text, resolve_program_id +from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, ReportedVerifier +from litellm.types.router import OracleRouterConfig + +MODELS = ("smart", "fast") + + +class _RecordingMaker: + """Alternates models per bind and records every update in arrival order.""" + + def __init__(self) -> None: + self.selections = 0 + self.updates: list[tuple[str, str, float]] = [] + + @property + def models(self): + return MODELS + + async def select(self, context: ProgramContext) -> str: + self.selections += 1 + return MODELS[(self.selections - 1) % 2] + + def update(self, context: ProgramContext, model: str, score: float) -> None: + self.updates.append((context.program_id, model, score)) + + def snapshot(self): + return {"type": "recording"} + + +class _SlowVerifier: + def __init__(self, delays: dict[str, float]) -> None: + self.delays = delays + + async def verify(self, outcome: ProgramOutcome) -> float: + await asyncio.sleep(self.delays.get(outcome.program_id, 0.0)) + return outcome.score if outcome.score is not None else 0.0 + + +def _router(verifier=None, maker=None, clock=None, **config) -> tuple[OracleRouter, _RecordingMaker]: + maker = maker or _RecordingMaker() + router = OracleRouter( + router_name="oracle", + config=OracleRouterConfig(available_models=list(MODELS), **config), + decision_maker=maker, + verifier=verifier or ReportedVerifier(), + **({"clock": clock} if clock else {}), + ) + return router, maker + + +def _messages(text: str = "fix the failing test in src/parser.py"): + return [{"role": "system", "content": "be terse"}, {"role": "user", "content": text}] + + +def test_resolve_program_id_precedence_and_header(): + assert resolve_program_id({"metadata": {"program_id": "a", "litellm_session_id": "b"}}, "program_id") == "a" + assert resolve_program_id({"metadata": {"litellm_session_id": "b"}}, "program_id") == "b" + assert resolve_program_id({"metadata": {"session_id": 7}}, "program_id") == "7" + assert resolve_program_id({"litellm_session_id": "lp"}, "program_id") == "lp" + assert resolve_program_id({"headers": {PROGRAM_ID_HEADER.upper(): "h"}}, "program_id") == "h" + assert resolve_program_id({"metadata": {"task": "t"}}, "task") == "t" + assert resolve_program_id({"metadata": {"program_id": ""}}, "program_id") is None + + +def test_first_user_text_reads_the_initial_request_including_text_parts(): + assert first_user_text(_messages("hello")) == "hello" + parts = [ + { + "role": "user", + "content": [{"type": "text", "text": "a"}, {"type": "image_url"}, {"type": "text", "text": "b"}], + } + ] + assert first_user_text(parts) == "a\nb" + assert first_user_text([{"role": "assistant", "content": "x"}]) == "" + assert first_user_text(None) == "" + + +@pytest.mark.asyncio +async def test_first_request_binds_and_later_requests_reuse_the_binding(): + router, maker = _router() + first = {"metadata": {"program_id": "t1", "user_api_key_hash": "key-1"}} + response = await router.async_pre_routing_hook(model="oracle", request_kwargs=first, messages=_messages()) + assert response is not None and response.model == "smart" + assert ( + first["metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart" and first["metadata"][PROGRAM_ID_METADATA_KEY] == "t1" + ) + assert response.routing_decision["router_type"] == "oracle" and response.routing_decision["routed_model"] == "smart" + + later = {"metadata": {"program_id": "t1"}} + again = await router.async_pre_routing_hook( + model="oracle", request_kwargs=later, messages=_messages("now run the tests") + ) + assert again is not None and again.model == "smart" + assert maker.selections == 1 + assert router.binding("t1").api_key_hash == "key-1" + + other = await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t2"}}, messages=_messages() + ) + assert other.model == "fast" and router.programs_bound == 2 + + +@pytest.mark.asyncio +async def test_requests_without_a_program_id_route_statelessly_and_never_learn(): + router, maker = _router() + kwargs = {"metadata": {}} + response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) + assert response.model == "smart" and kwargs["metadata"][PROGRAM_ID_METADATA_KEY] is None + assert router.stateless_requests == 1 and router.binding("") is None + assert router.complete("", score=1.0) is None + + +@pytest.mark.asyncio +async def test_complete_releases_the_slot_now_and_learns_in_verification_order(): + router, maker = _router(verifier=_SlowVerifier({"t1": 0.05, "t2": 0.0})) + for program_id in ("t1", "t2"): + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages() + ) + router.observe_request("t1", cost=0.4, response_text="patch") + router.observe_request("t1", cost=0.2, response_text="tests pass") + t1 = router.complete("t1", score=1.0) + t2 = router.complete("t2", score=0.0) + assert router.binding("t1") is None and router.binding("t2") is None + assert router.pending_verifications == 2 + assert await t2 == 0.0 + assert await t1 == 1.0 + assert [u[0] for u in maker.updates] == ["t2", "t1"] + assert maker.updates[1] == ("t1", "smart", 1.0) + snapshot = await router.get_state_snapshot() + assert snapshot["programs_completed"] == 2 and snapshot["recent_accuracy"] == {"smart": 1.0, "fast": 0.0} + assert snapshot["recent_feedback"][-1]["cost"] == pytest.approx(0.6) + + +@pytest.mark.asyncio +async def test_complete_cost_override_and_payload_reach_the_verifier(): + seen: list[ProgramOutcome] = [] + + class _Capture: + async def verify(self, outcome: ProgramOutcome) -> float: + seen.append(outcome) + return 1.0 + + router, maker = _router(verifier=_Capture()) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages("do x") + ) + transcript = [{"role": "user", "content": "do x"}, {"role": "assistant", "content": "answer"}] + router.observe_request("t", cost=0.3, response_text="answer", messages=transcript) + await router.complete("t", cost=2.5, payload={"container": "c1"}) + assert seen[0].cost == 2.5 and seen[0].payload == {"container": "c1"} and seen[0].response_text == "answer" + assert list(seen[0].messages) == transcript + assert seen[0].prompt == "do x" and maker.updates == [("t", "smart", 1.0)] + + +@pytest.mark.asyncio +async def test_failed_verification_is_counted_and_does_not_update(): + class _Broken: + async def verify(self, outcome): + raise RuntimeError("sandbox down") + + router, maker = _router(verifier=_Broken()) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + result = await router.complete("t") + assert result != result and router.verifications_failed == 1 and maker.updates == [] + + +@pytest.mark.asyncio +async def test_expired_programs_are_evicted_when_the_table_is_full(): + now = [1000.0] + router, maker = _router(clock=lambda: now[0], program_ttl_seconds=10, max_programs=3) + for program_id in ("a", "b", "c"): + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages() + ) + now[0] += 4.0 + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "d"}}, messages=_messages() + ) + assert router.binding("a") is None and router.binding("b") is not None and router.binding("d") is not None + assert router.programs_evicted == 1 + + +@pytest.mark.asyncio +async def test_decision_is_stamped_in_the_proxy_internal_bucket_when_one_exists(): + router, _ = _router() + kwargs = {"metadata": {"program_id": "t1"}, "litellm_metadata": {"user_api_key_hash": "k"}} + response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) + assert response.model == "smart" + assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart" + assert CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"] + assert router.binding("t1").api_key_hash == "k" diff --git a/tests/unit/router_strategy/oracle_router/test_router_dispatch.py b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py new file mode 100644 index 00000000000..36a3ad4059e --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py @@ -0,0 +1,125 @@ +"""Router-level wiring of `auto_router/oracle_router`: registration, deferred init, dispatch, hooks, removal.""" + +import pytest + +import litellm +from litellm import Router +from litellm.router_strategy.oracle_router.config import PROGRAM_ID_METADATA_KEY +from litellm.router_strategy.oracle_router.decision import PreRoutingDecisionMaker, ProgramContext +from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook +from litellm.types.router import LiteLLM_Params, RequestType + + +def _oracle(router: Router, name: str): + return router.oracle_routers[name][0].strategy + + +def _model_list(decision_maker=None, verifier=None, extra=()): + config = {"available_models": ["smart", "fast"]} + if decision_maker: + config["decision_maker"] = decision_maker + if verifier: + config["verifier"] = verifier + return [ + { + "model_name": "oracle", + "litellm_params": {"model": "auto_router/oracle_router", "oracle_router_config": config}, + }, + {"model_name": "smart", "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}}, + {"model_name": "fast", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}, + *extra, + ] + + +def _oracle_hooks(): + return litellm.logging_callback_manager.get_custom_loggers_for_type(OracleRouterPostCallHook) + + +def test_prefix_is_classified_as_oracle_and_not_as_a_semantic_router(): + router = Router(model_list=[]) + params = LiteLLM_Params(model="auto_router/oracle_router") + assert router._is_oracle_router_deployment(litellm_params=params) is True + assert router._is_auto_router_deployment(litellm_params=params) is False + assert router._is_adaptive_router_deployment(litellm_params=params) is False + assert router._is_oracle_router_deployment(litellm_params=LiteLLM_Params(model="openai/gpt-4o")) is False + + +def test_init_registers_the_router_and_exactly_one_hook(): + router = Router(model_list=_model_list()) + assert "oracle" in router.oracle_routers + assert _oracle(router, "oracle").models == ("smart", "fast") + hooks = [hook for hook in _oracle_hooks() if hook.oracle_router is _oracle(router, "oracle")] + assert len(hooks) == 1 + + +def test_init_without_config_raises(): + with pytest.raises(ValueError, match="oracle_router_config is required"): + Router(model_list=[{"model_name": "oracle", "litellm_params": {"model": "auto_router/oracle_router"}}]) + + +@pytest.mark.asyncio +async def test_pre_routing_hook_dispatches_to_the_oracle_router_and_binds_the_program(): + router = Router(model_list=_model_list(decision_maker={"type": "fixed", "model": "fast"})) + kwargs = {"metadata": {"program_id": "task-1"}} + response = await router.async_pre_routing_hook( + model="oracle", request_kwargs=kwargs, messages=[{"role": "user", "content": "fix the bug"}] + ) + assert response is not None and response.model == "fast" + assert response.routing_decision["router_type"] == "oracle" + assert kwargs["metadata"][PROGRAM_ID_METADATA_KEY] == "task-1" + assert _oracle(router, "oracle").binding("task-1").model == "fast" + + +def test_pre_routing_decision_maker_resolves_a_strategy_router_listed_after_it(): + quality = { + "model_name": "tiered", + "litellm_params": { + "model": "auto_router/quality_router", + "quality_router_default_model": "fast", + "quality_router_config": {"available_models": ["smart", "fast"]}, + }, + } + model_list = _model_list(decision_maker={"type": "pre_routing", "router": "tiered"}, extra=(quality,)) + model_list[1]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 3}} + model_list[2]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 1}} + router = Router(model_list=model_list) + assert isinstance(_oracle(router, "oracle").decision_maker, PreRoutingDecisionMaker) + + +def test_init_reads_adaptive_router_preferences_as_bandit_priors(): + model_list = _model_list() + model_list[1]["model_info"] = {"adaptive_router_preferences": {"quality_tier": 3, "strengths": []}} + router = Router(model_list=model_list) + maker = _oracle(router, "oracle").decision_maker + ctx = ProgramContext(program_id="p", prompt="x", request_type=RequestType.GENERAL) + assert maker.estimate(ctx, "smart") > maker.estimate(ctx, "fast") + + +def test_pre_routing_decision_maker_with_unknown_router_fails_init(): + with pytest.raises(ValueError, match="not a configured strategy router"): + Router(model_list=_model_list(decision_maker={"type": "pre_routing", "router": "ghost"})) + + +def test_hot_reload_keeps_the_existing_router_and_its_learned_state(): + router = Router(model_list=_model_list()) + before = _oracle(router, "oracle") + router.set_model_list(_model_list()) + assert _oracle(router, "oracle") is before + assert len([hook for hook in _oracle_hooks() if hook.oracle_router is before]) == 1 + + +def test_deleting_the_deployment_unregisters_the_router_and_its_hook(): + router = Router(model_list=_model_list()) + before = _oracle(router, "oracle") + router.delete_deployment(id=router.get_model_ids(model_name="oracle")[0]) + assert "oracle" not in router.oracle_routers + assert all(hook.oracle_router is not before for hook in _oracle_hooks()) + + +def test_strategy_router_dependencies_cover_available_models(): + from litellm.router_utils.auto_router_model_naming import strategy_router_dependencies + + deps = strategy_router_dependencies( + {"model": "auto_router/oracle_router", "oracle_router_config": {"available_models": ["smart", "fast"]}} + ) + assert {(dep.model_name, dep.role) for dep in deps} == {("smart", "tier"), ("fast", "tier")} diff --git a/tests/unit/router_strategy/oracle_router/test_verifier.py b/tests/unit/router_strategy/oracle_router/test_verifier.py new file mode 100644 index 00000000000..43fa2083a09 --- /dev/null +++ b/tests/unit/router_strategy/oracle_router/test_verifier.py @@ -0,0 +1,100 @@ +"""Verifiers: LiteLLM guardrails and reported scores become the number the decision maker learns from.""" + +import pytest + +from litellm.router_strategy.oracle_router.verifier import ( + GuardrailVerifier, + ProgramOutcome, + ReportedVerifier, + Verifier, + build_verifier, + response_text, +) +from litellm.types.router import OracleVerifierConfig +from litellm.types.utils import Choices, Message, ModelResponse + + +def _outcome(score=None, cost=0.0, answer="done", messages=()) -> ProgramOutcome: + return ProgramOutcome( + program_id="p", model="smart", prompt="task", response_text=answer, cost=cost, messages=messages, score=score + ) + + +@pytest.mark.asyncio +async def test_reported_verifier_uses_the_reported_score_or_its_default(): + assert await ReportedVerifier().verify(_outcome(score=0.8)) == 0.8 + assert await ReportedVerifier().verify(_outcome(score=4.0)) == 1.0 + assert await ReportedVerifier(default=0.25).verify(_outcome()) == 0.25 + + +class _Guardrail: + def __init__(self, blocks: bool) -> None: + self.blocks = blocks + self.seen: list[dict] = [] + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + self.seen.append({"inputs": inputs, "input_type": input_type, "request_data": request_data}) + if self.blocks: + raise ValueError("blocked") + return inputs + + +@pytest.mark.asyncio +async def test_guardrail_verifier_hands_the_judge_the_transcript_and_the_feedback_payload(): + passing, blocking = _Guardrail(blocks=False), _Guardrail(blocks=True) + lookup = {"ok": passing, "strict": blocking}.get + transcript = ({"role": "user", "content": "task"}, {"role": "assistant", "content": "final answer"}) + outcome = ProgramOutcome( + program_id="p", + model="smart", + prompt="task", + response_text="final answer", + cost=0.0, + messages=transcript, + payload={"reference": "42"}, + ) + assert await GuardrailVerifier("ok", lookup=lookup).verify(outcome) == 1.0 + assert passing.seen == [ + { + "inputs": {"texts": ["final answer"], "model": "smart", "structured_messages": list(transcript)}, + "input_type": "response", + "request_data": {"model": "smart", "metadata": {"program_id": "p", "reference": "42"}}, + } + ] + assert await GuardrailVerifier("strict", lookup=lookup).verify(outcome) == 0.0 + with pytest.raises(LookupError): + await GuardrailVerifier("missing", lookup=lookup).verify(outcome) + + +@pytest.mark.asyncio +async def test_guardrail_verifier_without_a_transcript_sends_the_initial_request(): + passing = _Guardrail(blocks=False) + await GuardrailVerifier("ok", lookup={"ok": passing}.get).verify(_outcome()) + assert passing.seen[0]["inputs"]["structured_messages"] == [{"role": "user", "content": "task"}] + + +def test_response_text_reads_chat_completions_only(): + assert response_text(ModelResponse(choices=[Choices(message=Message(content="hi", role="assistant"))])) == "hi" + assert response_text({"choices": []}) == "" + assert response_text(None) == "" + + +class CustomVerifier: + async def verify(self, outcome: ProgramOutcome) -> float: + return 0.5 + + +class NotAVerifier: + pass + + +@pytest.mark.asyncio +async def test_build_verifier_by_type(): + assert isinstance(build_verifier(OracleVerifierConfig(type="reported")), ReportedVerifier) + assert isinstance(build_verifier(OracleVerifierConfig(type="guardrail", guardrail="g")), GuardrailVerifier) + custom = build_verifier(OracleVerifierConfig(type="custom", path=f"{__name__}:CustomVerifier")) + assert isinstance(custom, Verifier) and await custom.verify(_outcome()) == 0.5 + with pytest.raises(TypeError): + build_verifier(OracleVerifierConfig(type="custom", path=f"{__name__}:NotAVerifier")) + with pytest.raises(ValueError, match=r"verifier\.guardrail"): + OracleVerifierConfig(type="guardrail") From b9d9c426bc630447c534e83368a125668df8d757 Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Sat, 3 Oct 2026 00:43:00 +0000 Subject: [PATCH 02/10] fix(oracle_router): never learn from an unverified program; bind concurrent first requests once - GuardrailVerifier reads the status the guardrail records for itself (standard_logging_guardrail_information) before falling back to how it raised. A judge that failed to respond (llm_as_a_judge fails open and returns normally) or did not run now raises, so the program is a failed verification instead of a pass that teaches the bandit a success; a judge with on_failure: log scores 0 when the answer fails; a GuardrailRaisedException without blocked_content and any non-guardrail exception propagate instead of counting as "blocked". - A NaN/inf score can no longer reach a bandit cell: clamp_score rejects it (counted in verifications_failed) and metadata.program_score=nan is treated as no score. - Concurrent first requests of one program wait for the decision in flight and share a single binding; eviction leaves room for the program being bound (max_programs was exceeded by one). - Regenerate ui/litellm-dashboard/src/lib/http/schema.d.ts for the two new endpoints (check-ui-api-types); direct tests for the six new Router helpers (router_code_coverage); README documents the verifier semantics. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- .../router_strategy/oracle_router/README.md | 9 +- .../router_strategy/oracle_router/hooks.py | 4 +- .../oracle_router/oracle_router.py | 22 ++- .../router_strategy/oracle_router/verifier.py | 102 ++++++++++-- .../oracle_router/test_hooks.py | 11 ++ .../oracle_router/test_oracle_router.py | 35 ++++ .../oracle_router/test_router_dispatch.py | 59 ++++++- .../oracle_router/test_verifier.py | 152 +++++++++++++++++- ui/litellm-dashboard/src/lib/http/schema.d.ts | 146 ++++++++++++++++- 9 files changed, 519 insertions(+), 21 deletions(-) diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index 7827e735323..ebdb9fe2851 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -19,7 +19,8 @@ model_list: ``` Clients send `metadata.program_id` (or `litellm_session_id`, or the `x-litellm-program-id` header) -with every request of a task. Requests without any id are routed statelessly and never learn. +with every request of a task. Requests without any id are routed statelessly and never learn. Concurrent +first requests of one program wait for the decision in flight, so a program is bound exactly once. ## Decision makers @@ -34,12 +35,14 @@ with every request of a task. Requests without any id are routed statelessly and | `type` | scores a finished program by | |---|---| -| `guardrail` | applying the LiteLLM guardrail named `guardrail` to the program's final turn, with the transcript as `structured_messages` and the feedback `payload` as `request_data["metadata"]`: pass = 1, blocked = 0. An `llm_as_a_judge` guardrail is the usual choice | +| `guardrail` | applying the LiteLLM guardrail named `guardrail` to the program's final turn, with the transcript as `structured_messages` and the feedback `payload` as `request_data["metadata"]`: pass = 1, blocked = 0 (read from the status the guardrail records, so an `llm_as_a_judge` with `on_failure: log` still scores 0 when the answer fails). A guardrail that could not evaluate the program, for example a judge whose model was unreachable, counts as a failed verification and teaches the decision maker nothing | | `reported` (default) | `metadata.program_score` on the last request, or `POST /oracle_router/feedback` | | `custom` | `path: pkg.mod:Factory`, called with no arguments; must expose `async verify(outcome)` | A verified score `s` adds `s` successes and `1 - s` failures to the cell of the model that served the -program. Updates are applied in verification-arrival order. +program. Updates are applied in verification-arrival order. A verifier that raises, or returns a score +that is not a finite number, counts as a failed verification (`verifications_failed` in the state) and +never reaches the decision maker. ## Finishing a program diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py index a02c664629c..56801bf4e5b 100644 --- a/litellm/router_strategy/oracle_router/hooks.py +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -5,6 +5,7 @@ when the request carried ``metadata.program_done`` it completes the program: the and the verifier runs in the background. Nothing here can fail a request. """ +import math from collections.abc import Mapping from typing import TYPE_CHECKING, Final @@ -33,9 +34,10 @@ def _score(metadata: Mapping[str, object]) -> float | None: if isinstance(raw, bool) or not isinstance(raw, (int, float, str)): return None try: - return float(raw) + score: Final = float(raw) except ValueError: return None + return score if math.isfinite(score) else None # a NaN would corrupt a bandit cell for good class OracleRouterPostCallHook(CustomLogger): diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index acad4d6599a..4cb223f19cc 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -151,6 +151,7 @@ class OracleRouter: "bandit" if config.decision_maker.type in _LEARNING_DECISION_MAKERS else "classifier_plugin" ) self._bindings: Final[dict[str, ProgramBinding]] = {} # mutable-ok: the lookup table + self._binding_in_flight: Final[dict[str, asyncio.Event]] = {} # mutable-ok: programs being bound right now self._tasks: Final[set[asyncio.Task[float]]] = set() # mutable-ok: in-flight verifications self._history: Final[deque[FeedbackRecord]] = deque(maxlen=FEEDBACK_HISTORY_SIZE) # mutable-ok: bounded log self._update_lock: Final = asyncio.Lock() @@ -177,17 +178,34 @@ class OracleRouter: ) for program_id in expired: self._bindings.pop(program_id, None) - overflow: Final = len(self._bindings) - self.config.max_programs + overflow: Final = len(self._bindings) + 1 - self.config.max_programs # +1: room for the program being bound oldest: Final = tuple(sorted(self._bindings, key=lambda pid: self._bindings[pid].bound_at))[: max(overflow, 0)] for program_id in oldest: self._bindings.pop(program_id, None) self.programs_evicted += len(expired) + len(oldest) async def bind(self, program_id: str, prompt: str, api_key_hash: str | None) -> ProgramBinding: - """Bind a new program, or return its existing row.""" + """Bind a new program, or return its existing row. + + Concurrent first requests of one program (a harness that fans out) wait for the decision in + flight instead of each asking the decision maker, so a program is bound exactly once. + """ + in_flight = self._binding_in_flight.get(program_id) # rebind-ok: re-read after each wait + while in_flight is not None: + await in_flight.wait() + in_flight = self._binding_in_flight.get(program_id) existing: Final = self._bindings.get(program_id) if existing is not None: return existing + event: Final = asyncio.Event() + self._binding_in_flight[program_id] = event + try: + return await self._bind(program_id, prompt, api_key_hash) + finally: + self._binding_in_flight.pop(program_id, None) + event.set() + + async def _bind(self, program_id: str, prompt: str, api_key_hash: str | None) -> ProgramBinding: now: Final = self._clock() if len(self._bindings) >= min(PROGRAM_SWEEP_THRESHOLD, self.config.max_programs): self._evict_expired(now) diff --git a/litellm/router_strategy/oracle_router/verifier.py b/litellm/router_strategy/oracle_router/verifier.py index f35a30c0ec9..5419a5000e6 100644 --- a/litellm/router_strategy/oracle_router/verifier.py +++ b/litellm/router_strategy/oracle_router/verifier.py @@ -5,12 +5,16 @@ program's final turn with its transcript as context (``llm_as_a_judge``, ``custo check), the score the client reported, and an import path for an operator's own class. """ +import math from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field from types import MappingProxyType from typing import Final, Protocol, runtime_checkable +from pydantic import TypeAdapter, ValidationError + import litellm +from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.llms.openai import AllMessageValues from litellm.types.router import OracleVerifierConfig @@ -19,6 +23,11 @@ from litellm.types.utils import GenericGuardrailAPIInputs, ModelResponse from .decision import load_object _EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) +_STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) +_STATUS_RECORDS: Final = TypeAdapter(Sequence[Mapping[str, object]]) +_STATUS_RECORD_KEY: Final[str] = "standard_logging_guardrail_information" +_BLOCKED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_intervened", "guardrail_flagged"}) +_UNVERIFIED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_failed_to_respond", "not_run"}) @dataclass(frozen=True, slots=True) @@ -41,7 +50,11 @@ class Verifier(Protocol): def clamp_score(value: float) -> float: - return min(max(float(value), 0.0), 1.0) + """Clamp a score to [0, 1]. A NaN/inf score raises: it would corrupt a bandit cell for good.""" + score: Final = float(value) + if not math.isfinite(score): + raise ValueError(f"score must be finite, got {score!r}") + return min(max(score, 0.0), 1.0) def response_text(response: object) -> str: @@ -69,6 +82,57 @@ def find_guardrail(name: str) -> CustomGuardrail | None: return next((guardrail for guardrail in guardrails if guardrail.guardrail_name == name), None) +def _status_records(bucket: object) -> Sequence[Mapping[str, object]]: + """The ``standard_logging_guardrail_information`` records of one metadata bucket, or none.""" + try: + return _STATUS_RECORDS.validate_python(_STR_MAPPING.validate_python(bucket).get(_STATUS_RECORD_KEY) or ()) + except ValidationError: + return () + + +def recorded_status(request_data: Mapping[str, object], guardrail_name: str) -> str | None: + """The last status ``guardrail_name`` recorded for itself in ``request_data`` while it ran, if any. + + Every first-party guardrail writes a ``standard_logging_guardrail_information`` record into the + request's metadata bucket (``success``, ``guardrail_intervened``, ``guardrail_failed_to_respond``, + ...), which is the only way to tell a judge that failed the answer from one that failed to answer. + """ + records: Final = ( + *_status_records(request_data.get("metadata")), + *_status_records(request_data.get("litellm_metadata")), + ) + statuses: Final = tuple( + str(record["guardrail_status"]) + for record in records + if record.get("guardrail_name") in (guardrail_name, None) and record.get("guardrail_status") + ) + return statuses[-1] if statuses else None + + +async def _apply( + guardrail: CustomGuardrail, + inputs: GenericGuardrailAPIInputs, + request_data: dict[str, object], # mutable-ok: the guardrail writes its status into it +) -> Exception | None: + """Run the guardrail; the exception it raised to signal its verdict, or ``None`` when it returned.""" + try: + await guardrail.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # upstream types request_data as a bare dict + inputs=inputs, request_data=request_data, input_type="response" + ) + except Exception as error: # noqa: BLE001 # a guardrail may signal its verdict by raising; the caller classifies it + return error + return None + + +def _is_http_block(error: Exception) -> bool: + """A guardrail without a status record that blocks by raising FastAPI's ``HTTPException``.""" + try: + from fastapi import HTTPException + except ImportError: # the SDK without the proxy extra: nothing can raise it + return False + return isinstance(error, HTTPException) + + class GuardrailVerifier: """Apply a configured LiteLLM guardrail to the program's final turn: it passes or it is blocked. @@ -76,6 +140,13 @@ class GuardrailVerifier: ``structured_messages`` and the feedback payload as ``request_data["metadata"]``, so an ``llm_as_a_judge`` guardrail judges the whole conversation and a ``custom_code`` guardrail can read whatever the harness posted with the feedback. + + The verdict is read from the status the guardrail records for itself (so a judge configured with + ``on_failure: log`` still scores 0 when the answer fails), and falls back to how it raised: a + ``GuardrailRaisedException`` with ``blocked_content`` or an ``HTTPException`` is a block. A + guardrail that could not evaluate the program (``guardrail_failed_to_respond``, an unreachable + backend) raises, so the program counts as a failed verification and teaches the decision maker + nothing instead of passing it. """ def __init__(self, guardrail_name: str, lookup: Callable[[str], CustomGuardrail | None] = find_guardrail) -> None: @@ -90,18 +161,25 @@ class GuardrailVerifier: inputs: Final = GenericGuardrailAPIInputs( texts=[outcome.response_text], model=outcome.model, structured_messages=transcript ) - try: - await guardrail.apply_guardrail( # pyright: ignore[reportUnknownMemberType] # upstream types request_data as a bare dict - inputs=inputs, - request_data={ - "model": outcome.model, - "metadata": {"program_id": outcome.program_id, **outcome.payload}, - }, - input_type="response", - ) - except Exception: # noqa: BLE001 # a guardrail signals "blocked" by raising its own exception type + request_data: Final[dict[str, object]] = { # mutable-ok: the guardrail records its status into it + "model": outcome.model, + "metadata": {"program_id": outcome.program_id, **outcome.payload}, + } + verdict_error: Final = await _apply(guardrail, inputs, request_data) + status: Final = recorded_status(request_data, self._name) + if status in _UNVERIFIED_STATUSES: + raise RuntimeError(f"guardrail {self._name!r} did not evaluate program {outcome.program_id!r}: {status}") + if status in _BLOCKED_STATUSES: return 0.0 - return 1.0 + if status == "success" or verdict_error is None: + return 1.0 + if isinstance(verdict_error, GuardrailRaisedException): + if verdict_error.blocked_content: + return 0.0 + raise verdict_error # the guardrail could not reach a verdict + if _is_http_block(verdict_error): + return 0.0 + raise verdict_error def build_verifier(config: OracleVerifierConfig) -> Verifier: diff --git a/tests/unit/router_strategy/oracle_router/test_hooks.py b/tests/unit/router_strategy/oracle_router/test_hooks.py index 08360e35f00..ebc329a8244 100644 --- a/tests/unit/router_strategy/oracle_router/test_hooks.py +++ b/tests/unit/router_strategy/oracle_router/test_hooks.py @@ -82,6 +82,17 @@ async def test_program_done_completes_the_program_with_the_reported_score(bound) assert maker.updates == [("smart", 1.0)] +@pytest.mark.asyncio +async def test_a_non_finite_program_score_is_treated_as_no_score(bound): + router, maker, hook = bound + await hook.async_log_success_event( + _kwargs(cost=0.05, program_done=True, program_score="nan"), _response(), None, None + ) + assert router.binding("t1") is None + await router.drain() + assert maker.updates == [("smart", 0.0)] # the reported verifier's default, never NaN + + @pytest.mark.asyncio async def test_failure_event_does_not_add_spend_but_still_honours_program_done(bound): router, maker, hook = bound diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index 4d2ab51a2c8..240f93a7fe4 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -112,6 +112,41 @@ async def test_first_request_binds_and_later_requests_reuse_the_binding(): assert other.model == "fast" and router.programs_bound == 2 +@pytest.mark.asyncio +async def test_concurrent_first_requests_of_one_program_share_a_single_binding(): + class _SlowMaker(_RecordingMaker): + async def select(self, context: ProgramContext) -> str: + await asyncio.sleep(0.01) # a decision maker that does I/O (pre_routing, custom) + return await super().select(context) + + router, maker = _router(maker=_SlowMaker()) + responses = await asyncio.gather( + *( + router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t1"}}, messages=_messages() + ) + for _ in range(3) + ) + ) + assert {response.model for response in responses} == {"smart"} + assert maker.selections == 1 and router.programs_bound == 1 and router.binding("t1").model == "smart" + + +@pytest.mark.asyncio +async def test_a_non_finite_verifier_score_is_a_failed_verification(): + class _NaN: + async def verify(self, outcome: ProgramOutcome) -> float: + return float("nan") + + router, maker = _router(verifier=_NaN()) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + result = await router.complete("t") + assert result != result and router.verifications_failed == 1 and maker.updates == [] + assert router.binding("t") is None # the slot was released all the same + + @pytest.mark.asyncio async def test_requests_without_a_program_id_route_statelessly_and_never_learn(): router, maker = _router() diff --git a/tests/unit/router_strategy/oracle_router/test_router_dispatch.py b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py index 36a3ad4059e..7a3d0d3f550 100644 --- a/tests/unit/router_strategy/oracle_router/test_router_dispatch.py +++ b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py @@ -7,7 +7,7 @@ from litellm import Router from litellm.router_strategy.oracle_router.config import PROGRAM_ID_METADATA_KEY from litellm.router_strategy.oracle_router.decision import PreRoutingDecisionMaker, ProgramContext from litellm.router_strategy.oracle_router.hooks import OracleRouterPostCallHook -from litellm.types.router import LiteLLM_Params, RequestType +from litellm.types.router import Deployment, LiteLLM_Params, RequestType def _oracle(router: Router, name: str): @@ -116,6 +116,63 @@ def test_deleting_the_deployment_unregisters_the_router_and_its_hook(): assert all(hook.oracle_router is not before for hook in _oracle_hooks()) +def test_deployment_input_costs_prefer_litellm_params_over_model_info_and_skip_unknown_models(): + model_list = _model_list() + model_list[1]["litellm_params"]["input_cost_per_token"] = 2e-6 + model_list[1]["model_info"] = {"input_cost_per_token": 9e-6} + model_list[2]["model_info"] = {"input_cost_per_token": 1e-7} + router = Router(model_list=model_list) + assert router._deployment_input_costs(["smart", "fast", "ghost"]) == {"smart": 2e-6, "fast": 1e-7} + + +def test_deployment_adaptive_prefs_read_model_info_where_declared(): + model_list = _model_list() + model_list[1]["model_info"] = {"adaptive_router_preferences": {"quality_tier": 3, "strengths": ["code_generation"]}} + router = Router(model_list=model_list) + prefs = router._deployment_adaptive_prefs(["smart", "fast", "ghost"]) + assert set(prefs) == {"smart"} and prefs["smart"].quality_tier == 3 + assert prefs["smart"].strengths == [RequestType.CODE_GENERATION] + + +def test_resolve_strategy_router_finds_registered_strategy_routers_only(): + quality = { + "model_name": "tiered", + "litellm_params": { + "model": "auto_router/quality_router", + "quality_router_default_model": "fast", + "quality_router_config": {"available_models": ["smart", "fast"]}, + }, + } + model_list = _model_list(extra=(quality,)) + model_list[1]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 3}} + model_list[2]["model_info"] = {"litellm_routing_preferences": {"quality_tier": 1}} + router = Router(model_list=model_list) + assert router._resolve_strategy_router("tiered") is router.quality_routers["tiered"][0].strategy + assert router._resolve_strategy_router("smart") is None and router._resolve_strategy_router("ghost") is None + + +def test_init_oracle_router_deployment_registers_the_router_and_finalize_is_idempotent(): + router = Router(model_list=_model_list()[1:]) # the two plain deployments, no oracle router yet + deployment = Deployment( + model_name="late-oracle", + litellm_params=LiteLLM_Params( + model="auto_router/oracle_router", oracle_router_config={"available_models": ["smart", "fast"]} + ), + ) + router.init_oracle_router_deployment(deployment=deployment) + oracle = _oracle(router, "late-oracle") + assert oracle.models == ("smart", "fast") and oracle.router_name == "late-oracle" + router._sync_oracle_router_hooks() + router._sync_oracle_router_hooks() + assert len([hook for hook in _oracle_hooks() if hook.oracle_router is oracle]) == 1 + router._finalize_oracle_router_if_configured() # nothing new in model_list: the registry is untouched + assert _oracle(router, "late-oracle") is oracle + with pytest.raises(ValueError, match="oracle_router_config is required"): + router.init_oracle_router_deployment( + deployment=Deployment(model_name="bare", litellm_params=LiteLLM_Params(model="auto_router/oracle_router")) + ) + + def test_strategy_router_dependencies_cover_available_models(): from litellm.router_utils.auto_router_model_naming import strategy_router_dependencies diff --git a/tests/unit/router_strategy/oracle_router/test_verifier.py b/tests/unit/router_strategy/oracle_router/test_verifier.py index 43fa2083a09..20931fa8b97 100644 --- a/tests/unit/router_strategy/oracle_router/test_verifier.py +++ b/tests/unit/router_strategy/oracle_router/test_verifier.py @@ -1,13 +1,18 @@ """Verifiers: LiteLLM guardrails and reported scores become the number the decision maker learns from.""" import pytest +from fastapi import HTTPException +from litellm.exceptions import GuardrailRaisedException +from litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge import LLMAsAJudgeGuardrail from litellm.router_strategy.oracle_router.verifier import ( GuardrailVerifier, ProgramOutcome, ReportedVerifier, Verifier, build_verifier, + clamp_score, + recorded_status, response_text, ) from litellm.types.router import OracleVerifierConfig @@ -28,6 +33,8 @@ async def test_reported_verifier_uses_the_reported_score_or_its_default(): class _Guardrail: + """A guardrail that blocks the way LiteLLM's do: by raising, with no status record.""" + def __init__(self, blocks: bool) -> None: self.blocks = blocks self.seen: list[dict] = [] @@ -35,7 +42,22 @@ class _Guardrail: async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): self.seen.append({"inputs": inputs, "input_type": input_type, "request_data": request_data}) if self.blocks: - raise ValueError("blocked") + raise GuardrailRaisedException(guardrail_name="strict", message="blocked", blocked_content=True) + return inputs + + +class _RecordingGuardrail: + """A guardrail that reports through LiteLLM's status record, like llm_as_a_judge, and may also raise.""" + + def __init__(self, status: str, error: Exception | None = None, name: str = "judge") -> None: + self.status, self.error, self.name = status, error, name + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + request_data["metadata"]["standard_logging_guardrail_information"] = [ + {"guardrail_name": self.name, "guardrail_status": self.status} + ] + if self.error is not None: + raise self.error return inputs @@ -66,6 +88,134 @@ async def test_guardrail_verifier_hands_the_judge_the_transcript_and_the_feedbac await GuardrailVerifier("missing", lookup=lookup).verify(outcome) +@pytest.mark.asyncio +async def test_guardrail_verifier_reads_the_recorded_verdict_before_the_exception(): + judge = { + "passed": _RecordingGuardrail("success", name="passed"), + "logged": _RecordingGuardrail("guardrail_intervened", name="logged"), # on_failure: log -> no exception + "blocked": _RecordingGuardrail( + "guardrail_intervened", HTTPException(status_code=422, detail="judge"), "blocked" + ), + "flagged": _RecordingGuardrail("guardrail_flagged", name="flagged"), + } + assert await GuardrailVerifier("passed", lookup=judge.get).verify(_outcome()) == 1.0 + assert await GuardrailVerifier("logged", lookup=judge.get).verify(_outcome()) == 0.0 + assert await GuardrailVerifier("blocked", lookup=judge.get).verify(_outcome()) == 0.0 + assert await GuardrailVerifier("flagged", lookup=judge.get).verify(_outcome()) == 0.0 + + +@pytest.mark.asyncio +async def test_guardrail_that_could_not_evaluate_is_a_failed_verification_not_a_pass(): + judge = { + "down": _RecordingGuardrail("guardrail_failed_to_respond", name="down"), # llm_as_a_judge fails open + "skipped": _RecordingGuardrail("not_run", name="skipped"), + "crashed": _RecordingGuardrail("success", RuntimeError("boom"), "crashed"), + } + with pytest.raises(RuntimeError, match="did not evaluate"): + await GuardrailVerifier("down", lookup=judge.get).verify(_outcome()) + with pytest.raises(RuntimeError, match="not_run"): + await GuardrailVerifier("skipped", lookup=judge.get).verify(_outcome()) + + class _NoVerdict(_Guardrail): + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + raise GuardrailRaisedException(guardrail_name="g", message="timeout", blocked_content=False) + + with pytest.raises(GuardrailRaisedException): + await GuardrailVerifier("g", lookup={"g": _NoVerdict(blocks=True)}.get).verify(_outcome()) + assert await GuardrailVerifier("crashed", lookup=judge.get).verify(_outcome()) == 1.0 # the record wins + + +@pytest.mark.asyncio +async def test_guardrail_exceptions_without_a_record_are_classified(): + class _Raises: + def __init__(self, error: Exception) -> None: + self.error = error + + async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): + raise self.error + + lookup = { + "http": _Raises(HTTPException(status_code=400, detail="Violated guardrail policy")), + "infra": _Raises(ConnectionError("provider down")), + }.get + assert await GuardrailVerifier("http", lookup=lookup).verify(_outcome()) == 0.0 + with pytest.raises(ConnectionError): + await GuardrailVerifier("infra", lookup=lookup).verify(_outcome()) + + +def _judge(monkeypatch, on_failure: str, verdict) -> LLMAsAJudgeGuardrail: + """The real llm_as_a_judge guardrail with only its judge-model call stubbed.""" + guardrail = LLMAsAJudgeGuardrail( + guardrail_name="answer-judge", + judge_model="judge", + criteria=[{"name": "task_completed", "weight": 100, "description": "the task was completed"}], + overall_threshold=50, + on_failure=on_failure, + ) + + async def run_judge(messages, text_under_review, input_type="response"): + if isinstance(verdict, Exception): + raise verdict + return verdict + + monkeypatch.setattr(guardrail, "_run_judge", run_judge) + return guardrail + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("on_failure", "overall_score", "expected"), + [ + ("block", 90, 1.0), # passes the threshold: recorded as success + ("block", 10, 0.0), # fails: the judge raises 422 and records guardrail_intervened + ("log", 10, 0.0), # fails but only logs: no exception, still recorded as guardrail_intervened + ], +) +async def test_real_llm_as_a_judge_verdicts_become_scores(monkeypatch, on_failure, overall_score, expected): + guardrail = _judge(monkeypatch, on_failure, {"overall_score": overall_score, "verdicts": []}) + verifier = GuardrailVerifier("answer-judge", lookup={"answer-judge": guardrail}.get) + assert await verifier.verify(_outcome(answer="final answer")) == expected + + +@pytest.mark.asyncio +async def test_real_llm_as_a_judge_that_fails_open_is_not_a_pass(monkeypatch): + guardrail = _judge(monkeypatch, "block", RuntimeError("judge model unreachable")) # the guardrail returns normally + verifier = GuardrailVerifier("answer-judge", lookup={"answer-judge": guardrail}.get) + with pytest.raises(RuntimeError, match="did not evaluate"): + await verifier.verify(_outcome(answer="final answer")) + + +def test_recorded_status_reads_both_buckets_and_takes_the_latest(): + data = { + "metadata": { + "standard_logging_guardrail_information": [{"guardrail_name": "a", "guardrail_status": "success"}] + }, + "litellm_metadata": { + "standard_logging_guardrail_information": {"guardrail_name": "a", "guardrail_status": "guardrail_flagged"} + }, + } + assert recorded_status(data, "a") == "success" # a single dict, not a list, is not a valid record set + data["litellm_metadata"]["standard_logging_guardrail_information"] = [ + {"guardrail_name": "a", "guardrail_status": "guardrail_intervened"} + ] + assert recorded_status(data, "a") == "guardrail_intervened" + assert recorded_status(data, "other") is None + assert recorded_status({"metadata": "junk"}, "a") is None + + +def test_clamp_score_rejects_non_finite_scores(): + assert clamp_score(1.7) == 1.0 and clamp_score(-2) == 0.0 + for bad in (float("nan"), float("inf")): + with pytest.raises(ValueError, match="finite"): + clamp_score(bad) + + +@pytest.mark.asyncio +async def test_reported_verifier_rejects_a_nan_score(): + with pytest.raises(ValueError, match="finite"): + await ReportedVerifier().verify(_outcome(score=float("nan"))) + + @pytest.mark.asyncio async def test_guardrail_verifier_without_a_transcript_sends_the_initial_request(): passing = _Guardrail(blocks=False) diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index a376f8d7910..ebb7ffc5840 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11380,6 +11380,49 @@ export interface paths { patch: operations["openrouter_proxy_route_openrouter__endpoint__patch"]; trace?: never; }; + "/oracle_router/feedback": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Submit Oracle Router Feedback + * @description Complete a program: its model slot is released now, its verifier runs in the background. + * + * The key that started the program (or an admin) may complete it. 404 when no ORACLE router holds a + * program with that id, which also covers programs completed earlier. + */ + post: operations["submit_oracle_router_feedback_oracle_router_feedback_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/oracle_router/state": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Get Oracle Router State + * @description Decision-maker state, active bindings and recent verified feedback for every ORACLE router. Admin only. + */ + get: operations["get_oracle_router_state_oracle_router_state_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/organization/daily/activity": { parameters: { query?: never; @@ -35044,6 +35087,10 @@ export interface components { ocr_cost_per_page?: number | null; /** Ocr Cost Per Page Batches */ ocr_cost_per_page_batches?: number | null; + /** Oracle Router Config */ + oracle_router_config?: { + [key: string]: unknown; + } | null; /** Organization */ organization?: string | null; /** Otpm */ @@ -39238,6 +39285,46 @@ export interface components { } & { [key: string]: unknown; }; + /** + * OracleRouterFeedbackRequest + * @description Report that a program finished. ``score`` is optional when the router's verifier grades it. + */ + OracleRouterFeedbackRequest: { + /** + * Cost + * @description Overrides the spend LiteLLM accumulated for the program + */ + cost?: number | null; + /** + * Payload + * @description Passed to the verifier unchanged + */ + payload?: { + [key: string]: unknown; + }; + /** Program Id */ + program_id: string; + /** Score */ + score?: number | null; + }; + /** OracleRouterFeedbackResponse */ + OracleRouterFeedbackResponse: { + /** Model */ + model: string; + /** Pending Verifications */ + pending_verifications: number; + /** Program Id */ + program_id: string; + /** Router Name */ + router_name: string; + }; + /** OracleRouterStateResponse */ + OracleRouterStateResponse: { + /** Routers */ + routers: { + [key: string]: unknown; + }[]; + }; /** OrgMember */ OrgMember: { /** @@ -45272,7 +45359,7 @@ export interface components { * Router Type * @enum {string} */ - router_type?: "complexity" | "adaptive" | "quality"; + router_type?: "complexity" | "adaptive" | "quality" | "oracle"; /** Savings Baseline Deployment Id */ savings_baseline_deployment_id?: string; /** Savings Baseline Model */ @@ -49685,6 +49772,10 @@ export interface components { ocr_cost_per_page?: number | null; /** Ocr Cost Per Page Batches */ ocr_cost_per_page_batches?: number | null; + /** Oracle Router Config */ + oracle_router_config?: { + [key: string]: unknown; + } | null; /** Organization */ organization?: string | null; /** Otpm */ @@ -65833,6 +65924,59 @@ export interface operations { }; }; }; + submit_oracle_router_feedback_oracle_router_feedback_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["OracleRouterFeedbackRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["OracleRouterFeedbackResponse"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_oracle_router_state_oracle_router_state_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["OracleRouterStateResponse"]; + }; + }; + }; + }; get_organization_daily_activity_organization_daily_activity_get: { parameters: { query?: { From aded7e7c0dfc8275a176618ba9e40169ed7890df Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Sat, 3 Oct 2026 01:03:55 +0000 Subject: [PATCH 03/10] fix(router): type the oracle-router helpers; build the router on first upsert - upsert_deployment dereferenced the deployment previously on the router to decide whether to finalize the oracle router, so upserting a NEW oracle deployment (the proxy's DB reload path) raised AttributeError instead of building it. Decide from the deployment being upserted. - _finalize_oracle_router_if_configured, _deployment_input_costs and _deployment_adaptive_prefs read deployments through the typed accessors (get_deployment, get_deployment_by_model_group_name, model_info fields and extras) instead of scanning the untyped model_list, and the config is built with model_validate. The finalize loop no longer scans model_list directly, which test_no_linear_scans_in_router rejects for methods outside its allow list. - The adaptive router's finalize loop and init read deployments through the same accessors and the two shared helpers instead of their own dict scan, so its allow-list entry goes away. Net basedpyright effect against main: reportUnknownArgumentType 44906 -> 44792, which absorbs the one error per **kwargs call site that the new litellm_params field adds (111) without a limit bump. - clear_cache: db_router_names is a set[str], so every strategy-registry pop on it is typed. - Test: upserting an oracle deployment builds its router once, leaves it untouched when unchanged, and rebuilds it when its config changes. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- .../model_management_endpoints.py | 4 +- litellm/router.py | 142 ++++++------------ .../test_router_index_management.py | 1 - .../oracle_router/test_router_dispatch.py | 29 ++++ 4 files changed, 76 insertions(+), 100 deletions(-) diff --git a/litellm/proxy/management_endpoints/model_management_endpoints.py b/litellm/proxy/management_endpoints/model_management_endpoints.py index 0596946f44e..b9f9fa61bfa 100644 --- a/litellm/proxy/management_endpoints/model_management_endpoints.py +++ b/litellm/proxy/management_endpoints/model_management_endpoints.py @@ -3264,7 +3264,7 @@ async def clear_cache() -> ReconcileOutcome: config_models: Final = [] db_model_ids: Final = [] - db_router_names: Final = set() + db_router_names: Final[set[str]] = set() # mutable-ok: filled in the classification pass below for model in current_models: model_info = model.get("model_info", {}) @@ -3273,7 +3273,7 @@ async def clear_cache() -> ReconcileOutcome: # Auto-router deployments (and only those) are wiped here, in the # same pass, so the reload rebuilds them -- see the comment below. model_name = model.get("model_name") - if model_name is not None and str(model.get("litellm_params", {}).get("model", "")).startswith( + if isinstance(model_name, str) and str(model.get("litellm_params", {}).get("model", "")).startswith( "auto_router/" ): db_router_names.add(model_name) diff --git a/litellm/router.py b/litellm/router.py index 9af3fc2fab6..6397e9eeef9 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9319,20 +9319,13 @@ class Router: build an AdaptiveRouter for each. Safe no-op when none are configured. Idempotent: skips any deployment whose (model_name, tags) pair is already initialized, so hot-reloads don't rebuild routers that would lose state.""" - for entry in self.model_list or []: - lp = entry.get("litellm_params") if isinstance(entry, dict) else entry.litellm_params - lp_model = (lp.get("model") if isinstance(lp, dict) else lp.model) if lp else None - if not (lp_model and lp_model.startswith("auto_router/adaptive_router")): + for model_id in self.get_model_ids(): + deployment = self.get_deployment(model_id=model_id) + if deployment is None or not self._is_adaptive_router_deployment(litellm_params=deployment.litellm_params): continue - model_name = entry.get("model_name") if isinstance(entry, dict) else entry.model_name - if not model_name or not lp: - continue - deployment = Deployment( - model_name=model_name, - litellm_params=(lp if not isinstance(lp, dict) else LiteLLM_Params(**lp)), - model_info=(entry.get("model_info") if isinstance(entry, dict) else entry.model_info), - ) - if self._has_registered_strategy(self.adaptive_routers, model_name, self._deployment_tags(deployment)): + if self._has_registered_strategy( + self.adaptive_routers, deployment.model_name, self._deployment_tags(deployment) + ): continue self.init_adaptive_router_deployment(deployment=deployment) @@ -9368,6 +9361,34 @@ class Router: AdaptiveRouterPostCallHook(adaptive_router=tagged.strategy) ) + def _deployment_input_costs( + self, model_names: Sequence[str] + ) -> dict[str, float]: # mutable-ok: pick_best takes a dict + """``input_cost_per_token`` of the first deployment of each model name; litellm_params wins over model_info. + + Bandit priors for the adaptive and oracle routers, keyed by ``available_models`` entry.""" + costs: Final[dict[str, float]] = {} # mutable-ok: built per model name in the loop below + for name in model_names: + deployment = self.get_deployment_by_model_group_name(model_group_name=name) + # Deployment.__init__ mirrors a litellm_params rate onto model_info, so one field reads both. + cost = deployment.model_info.input_cost_per_token if deployment is not None else None + if cost is not None: + costs[name] = cost + return costs + + def _deployment_adaptive_prefs( + self, model_names: Sequence[str] + ) -> dict[str, AdaptiveRouterPreferences]: # mutable-ok: consumed as a mapping by the bandit + """``model_info.adaptive_router_preferences`` of the first deployment of each model name, where declared.""" + prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} # mutable-ok: built per model name in the loop below + for name in model_names: + deployment = self.get_deployment_by_model_group_name(model_group_name=name) + declared = deployment.model_info.model_extra if deployment is not None else None + raw = declared.get("adaptive_router_preferences") if declared else None + if raw is not None: + prefs[name] = AdaptiveRouterPreferences.model_validate(raw) + return prefs + def init_adaptive_router_deployment(self, deployment: Deployment) -> None: """ Build an AdaptiveRouter instance for this deployment and register its @@ -9386,46 +9407,18 @@ class Router: from litellm.router_strategy.adaptive_router.hooks import ( AdaptiveRouterPostCallHook, ) - from litellm.types.router import ( - AdaptiveRouterConfig, - AdaptiveRouterPreferences, - ) + from litellm.types.router import AdaptiveRouterConfig raw_config: Final = deployment.litellm_params.adaptive_router_config if raw_config is None: raise ValueError("adaptive_router_config is required for adaptive-router deployments.") config: Final = AdaptiveRouterConfig(**raw_config) - - model_to_prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} - model_to_cost: Final[dict[str, float]] = {} - # O(k) via the name→indices map: only touch deployments whose name - # is listed in `available_models`, instead of scanning model_list. - for name in config.available_models: - indices = self.model_name_to_deployment_indices.get(name, []) - if not indices: - continue - d = (self.model_list or [])[indices[0]] - mi = d.get("model_info") if isinstance(d, dict) else d.model_info - mi_dict: dict[str, Any] = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) - prefs_raw = mi_dict.get("adaptive_router_preferences") - if prefs_raw is not None: - model_to_prefs[name] = AdaptiveRouterPreferences(**prefs_raw) - - # model_info is the conventional pricing location elsewhere in LiteLLM; litellm_params wins if set. - lp = d.get("litellm_params") if isinstance(d, dict) else d.litellm_params - lp_dict: dict[str, Any] = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) - cost = lp_dict.get("input_cost_per_token") - if cost is None: - cost = mi_dict.get("input_cost_per_token") - if cost is not None: - model_to_cost[name] = float(cost) - adaptive_router: Final = AdaptiveRouter( router_name=deployment.model_name, config=config, - model_to_prefs=model_to_prefs, - model_to_cost=model_to_cost, + model_to_prefs=self._deployment_adaptive_prefs(config.available_models), + model_to_cost=self._deployment_input_costs(config.available_models), ) self._register_pre_routing_strategy( registry=self.adaptive_routers, @@ -9452,20 +9445,13 @@ class Router: Deferred like the adaptive router: a `pre_routing` decision maker wraps another strategy router that may appear later in the list. Idempotent across hot reloads, so a router that already holds program bindings and learned state is never rebuilt.""" - for entry in self.model_list or []: - lp = entry.get("litellm_params") if isinstance(entry, dict) else entry.litellm_params - lp_model = (lp.get("model") if isinstance(lp, dict) else lp.model) if lp else None - if not (lp_model and classify_strategy_router_model(lp_model) == "oracle"): + for model_id in self.get_model_ids(): + deployment = self.get_deployment(model_id=model_id) + if deployment is None or not self._is_oracle_router_deployment(litellm_params=deployment.litellm_params): continue - model_name = entry.get("model_name") if isinstance(entry, dict) else entry.model_name - if not model_name or not lp: - continue - deployment = Deployment( - model_name=model_name, - litellm_params=(lp if not isinstance(lp, dict) else LiteLLM_Params(**lp)), - model_info=(entry.get("model_info") if isinstance(entry, dict) else entry.model_info), - ) - if self._has_registered_strategy(self.oracle_routers, model_name, self._deployment_tags(deployment)): + if self._has_registered_strategy( + self.oracle_routers, deployment.model_name, self._deployment_tags(deployment) + ): continue self.init_oracle_router_deployment(deployment=deployment) self._sync_oracle_router_hooks() @@ -9482,44 +9468,6 @@ class Router: OracleRouterPostCallHook(oracle_router=tagged.strategy) ) - def _deployment_input_costs( - self, model_names: Sequence[str] - ) -> dict[str, float]: # mutable-ok: pick_best takes a dict - """``input_cost_per_token`` of the first deployment of each model name, from litellm_params or model_info.""" - costs: Final[dict[str, float]] = {} # mutable-ok: built per model name in the loop below - for name in model_names: - indices = self.model_name_to_deployment_indices.get(name, []) - if not indices: - continue - entry = (self.model_list or [])[indices[0]] - lp = entry.get("litellm_params") if isinstance(entry, dict) else entry.litellm_params - lp_dict = lp if isinstance(lp, dict) else (lp.model_dump() if lp else {}) - mi = entry.get("model_info") if isinstance(entry, dict) else entry.model_info - mi_dict = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) - cost = lp_dict.get("input_cost_per_token") - if cost is None: - cost = mi_dict.get("input_cost_per_token") - if cost is not None: - costs[name] = float(cost) - return costs - - def _deployment_adaptive_prefs( - self, model_names: Sequence[str] - ) -> dict[str, AdaptiveRouterPreferences]: # mutable-ok: consumed as a mapping by the bandit - """``model_info.adaptive_router_preferences`` of the first deployment of each model name, where declared.""" - prefs: Final[dict[str, AdaptiveRouterPreferences]] = {} # mutable-ok: built per model name in the loop below - for name in model_names: - indices = self.model_name_to_deployment_indices.get(name, []) - if not indices: - continue - entry = (self.model_list or [])[indices[0]] - mi = entry.get("model_info") if isinstance(entry, dict) else entry.model_info - mi_dict = mi if isinstance(mi, dict) else (mi.model_dump() if mi else {}) - raw = mi_dict.get("adaptive_router_preferences") - if raw is not None: - prefs[name] = AdaptiveRouterPreferences(**raw) - return prefs - def _resolve_strategy_router(self, model_name: str) -> "PreRoutingStrategy | None": """The first strategy router registered under `model_name`, for ORACLE's `pre_routing` decision maker.""" for registry in (self.complexity_routers, self.quality_routers, self.adaptive_routers, self.auto_routers): @@ -9538,7 +9486,7 @@ class Router: raw_config: Final = deployment.litellm_params.oracle_router_config if raw_config is None: raise ValueError("oracle_router_config is required for oracle-router deployments.") - config: Final = OracleRouterConfig(**raw_config) + config: Final = OracleRouterConfig.model_validate(raw_config) oracle_router: Final = OracleRouter( router_name=deployment.model_name, config=config, @@ -10088,7 +10036,7 @@ class Router: ) ): self._finalize_adaptive_router_if_configured() - if self._is_oracle_router_deployment(litellm_params=_deployment_on_router.litellm_params): + if self._is_oracle_router_deployment(litellm_params=deployment.litellm_params): self._finalize_oracle_router_if_configured() return deployment except Exception as e: diff --git a/tests/router_unit_tests/test_router_index_management.py b/tests/router_unit_tests/test_router_index_management.py index 291badd91b4..d389513b57a 100644 --- a/tests/router_unit_tests/test_router_index_management.py +++ b/tests/router_unit_tests/test_router_index_management.py @@ -239,7 +239,6 @@ class TestRouterIndexManagement: # Methods that are allowed to iterate through self.model_list ALLOWED_METHODS = { "_get_deployment_by_litellm_model": "lookup by litellm_params.model, which is not indexed", - "_finalize_adaptive_router_if_configured": 'init-time prefix scan for "auto_router/adaptive_router"; no index for prefix match', "config_deployments": "filters the whole list on model_info.db_model; admin path only (model add/upsert)", "auto_router_capability_violation": "counts gated auto-routers across the whole list; admin path only (auto-router init/upsert)", } diff --git a/tests/unit/router_strategy/oracle_router/test_router_dispatch.py b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py index 7a3d0d3f550..6e6d026606b 100644 --- a/tests/unit/router_strategy/oracle_router/test_router_dispatch.py +++ b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py @@ -173,6 +173,35 @@ def test_init_oracle_router_deployment_registers_the_router_and_finalize_is_idem ) +def test_upserting_an_oracle_deployment_builds_its_router_once_and_rebuilds_it_when_its_config_changes(): + router = Router(model_list=_model_list()[1:]) # the two plain deployments, no oracle router yet + deployment = Deployment( + model_name="late-oracle", + litellm_params=LiteLLM_Params( + model="auto_router/oracle_router", oracle_router_config={"available_models": ["smart", "fast"]} + ), + model_info={"id": "oracle-1"}, + ) + router.upsert_deployment(deployment=deployment) # a new id: nothing was on the router before + oracle = _oracle(router, "late-oracle") + assert oracle.models == ("smart", "fast") + router.upsert_deployment(deployment=deployment) # unchanged: the router and its learned state survive + assert _oracle(router, "late-oracle") is oracle + assert len([hook for hook in _oracle_hooks() if hook.oracle_router is oracle]) == 1 + router.upsert_deployment( + deployment=Deployment( + model_name="late-oracle", + litellm_params=LiteLLM_Params( + model="auto_router/oracle_router", oracle_router_config={"available_models": ["fast"]} + ), + model_info={"id": "oracle-1"}, + ) + ) + rebuilt = _oracle(router, "late-oracle") + assert rebuilt is not oracle and rebuilt.models == ("fast",) + assert [hook.oracle_router for hook in _oracle_hooks() if hook.oracle_router in (oracle, rebuilt)] == [rebuilt] + + def test_strategy_router_dependencies_cover_available_models(): from litellm.router_utils.auto_router_model_naming import strategy_router_dependencies From d14093db53f5a186cff2367b263288b283d1d855 Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Sat, 3 Oct 2026 17:50:13 +0000 Subject: [PATCH 04/10] fix(oracle_router): key-scoped programs, trusted stamps, config-only custom plugins Security review findings on #44290: - Custom decision makers and verifiers import and call Python named by the deployment. A model written through the API (/model/new, /model/update) with `type: custom` is now rejected when the router is built, before any import, whatever the caller's role; the config file is the only place that may name plugin code. (High: a team admin could run `subprocess:Popen` through a DB-backed deployment during reload.) - A program is private to the API key that started it (user_api_key_hash, stamped server-side): the same program_id under another key is another program, so no caller can ride on, observe or complete someone else's binding. /oracle_router/feedback resolves the caller's own program; an admin may complete any key's. A stranger's lookup is a 404, not a 403, so program ids leak nothing. - The post-call hook (spend, transcript, program_done completion) and the response-header hook only act on requests the Router stamped as routed by this oracle router (routing_decision, which the Router writes or clears on every attempt), and only against the program of the key that sent them. The pre-routing hook drops any caller-supplied copy of its internal metadata keys from both buckets before stamping its own. (Medium: a chat request to any model carrying oracle_router_program_id + program_done could terminate another program and inject its score.) CI: - /oracle_router/feedback and /oracle_router/state join the gateway route allowlist: the program bindings live in the serving process, and test_gateway_plus_backend_covers_full_app (proxy-infra-root) rejects any route no component serves. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- gateway/routes/allowlist.py | 2 + .../oracle_router_endpoints.py | 36 +++++-- litellm/router.py | 7 ++ .../router_strategy/oracle_router/README.md | 13 +++ .../router_strategy/oracle_router/hooks.py | 41 +++++-- .../oracle_router/oracle_router.py | 100 ++++++++++++------ litellm/types/router.py | 6 +- .../test_oracle_router_endpoints.py | 25 +++-- .../oracle_router/test_hooks.py | 49 ++++++++- .../oracle_router/test_oracle_router.py | 42 +++++++- .../oracle_router/test_router_dispatch.py | 45 ++++++++ 11 files changed, 299 insertions(+), 67 deletions(-) diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 6e91f5486d0..e3f1af247b4 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -116,6 +116,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( # Realtime / streaming "/v1/realtime", "/realtime", + # ORACLE router feedback and state: the program bindings live in the serving process + "/oracle_router/", # Health & ops "/health", "/metrics", diff --git a/litellm/proxy/management_endpoints/oracle_router_endpoints.py b/litellm/proxy/management_endpoints/oracle_router_endpoints.py index 27e79fe6785..a88bab1fe6e 100644 --- a/litellm/proxy/management_endpoints/oracle_router_endpoints.py +++ b/litellm/proxy/management_endpoints/oracle_router_endpoints.py @@ -19,7 +19,7 @@ from litellm.types.management_endpoints.oracle_router_endpoints import ( if TYPE_CHECKING: from fastapi import APIRouter, Depends, HTTPException - from litellm.router_strategy.oracle_router.oracle_router import OracleRouter + from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, ProgramBinding else: try: from fastapi import APIRouter, Depends, HTTPException @@ -41,6 +41,22 @@ def _oracle_routers() -> tuple["OracleRouter", ...]: return tuple(tagged.strategy for tagged in tagged_entries) +def _find_program( + routers: tuple["OracleRouter", ...], program_id: str, user_api_key_dict: UserAPIKeyAuth +) -> tuple["OracleRouter", "ProgramBinding"] | None: + """The caller's program with this id. An admin may also reach another key's program of that id.""" + for candidate in routers: + owned = candidate.binding(program_id, user_api_key_dict.api_key) + if owned is not None: + return candidate, owned + if not _is_admin(user_api_key_dict): + return None + for candidate in routers: + for any_owner in candidate.bindings_named(program_id): + return candidate, any_owner + return None + + @router.post( "/oracle_router/feedback", tags=["oracle_router"], @@ -53,26 +69,24 @@ async def submit_oracle_router_feedback( ) -> OracleRouterFeedbackResponse: """Complete a program: its model slot is released now, its verifier runs in the background. - The key that started the program (or an admin) may complete it. 404 when no ORACLE router holds a - program with that id, which also covers programs completed earlier. + Programs are private to the key that started them; an admin may complete any key's program. 404 when + the caller holds no program with that id, which also covers programs completed earlier. """ routers: Final = _oracle_routers() if not routers: raise HTTPException(status_code=404, detail={"error": "No oracle_router is configured on this proxy."}) - owner: Final = next((candidate for candidate in routers if candidate.binding(data.program_id) is not None), None) - binding: Final = owner.binding(data.program_id) if owner is not None else None - if owner is None or binding is None: + found: Final = _find_program(routers, data.program_id, user_api_key_dict) + if found is None: raise HTTPException( status_code=404, detail={"error": f"Unknown or already completed program {data.program_id!r}."} ) - if binding.api_key_hash and binding.api_key_hash != user_api_key_dict.api_key and not _is_admin(user_api_key_dict): - raise HTTPException(status_code=403, detail={"error": CommonProxyErrors.not_allowed_access.value}) - owner.complete(data.program_id, score=data.score, cost=data.cost, payload=data.payload) + holder, binding = found + holder.complete(data.program_id, score=data.score, cost=data.cost, payload=data.payload, owner=binding.api_key_hash) return OracleRouterFeedbackResponse( program_id=data.program_id, - router_name=owner.router_name, + router_name=holder.router_name, model=binding.model, - pending_verifications=owner.pending_verifications, + pending_verifications=holder.pending_verifications, ) diff --git a/litellm/router.py b/litellm/router.py index 6397e9eeef9..194ae94f28e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9487,6 +9487,13 @@ class Router: if raw_config is None: raise ValueError("oracle_router_config is required for oracle-router deployments.") config: Final = OracleRouterConfig.model_validate(raw_config) + if deployment.model_info.db_model and "custom" in (config.decision_maker.type, config.verifier.type): + # A custom decision maker or verifier imports and runs Python named by the deployment. Only the + # operator's config file may name it; a model written through the API (any role) must not. + raise ValueError( + f"oracle_router deployment {deployment.model_name!r}: custom decision makers and verifiers " + "can only be configured in the proxy config file, not through the model API." + ) oracle_router: Final = OracleRouter( router_name=deployment.model_name, config=config, diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index ebdb9fe2851..9d05c3ae81e 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -51,6 +51,19 @@ verifier), or call `POST /oracle_router/feedback {"program_id": ..., "score": .. the harness once the task is graded. `GET /oracle_router/state` (admin) shows bindings, decision-maker state and recent feedback. The response header `x-litellm-oracle-router-model` names the bound model. +## Security + +A program is private to the API key that started it: the same `program_id` under another key is another +program, so no caller can ride on, observe or complete someone else's binding. `POST /oracle_router/feedback` +completes the caller's own program; an admin key may complete any key's. The post-call hook only reads +requests the Router stamped as routed by this router (`routing_decision`), so ORACLE keys placed in the +metadata of a request to some other model are ignored, and the router drops any caller-supplied copy of +its internal metadata keys before stamping its own. + +`custom` decision makers and verifiers import Python named by the deployment, so they are honoured only +for deployments in the proxy config file. A model written through the API (`/model/new`, `/model/update`, +whatever the caller's role) with a `custom` type is rejected when the router is built and never imported. + ## Limits State is in memory per proxy process; it is not persisted or shared across replicas. The paper's diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py index 56801bf4e5b..8ad8bded760 100644 --- a/litellm/router_strategy/oracle_router/hooks.py +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -2,7 +2,8 @@ After every completion it adds LiteLLM's computed spend and the latest answer to the program's row, and when the request carried ``metadata.program_done`` it completes the program: the slot is released at once -and the verifier runs in the background. Nothing here can fail a request. +and the verifier runs in the background. Only requests the Router stamped as routed by this ORACLE router +are read, and only against the program of the key that sent them. Nothing here can fail a request. """ import math @@ -26,7 +27,29 @@ if TYPE_CHECKING: def _request_metadata(kwargs: Mapping[str, object]) -> Mapping[str, object]: - return as_str_mapping(as_str_mapping(kwargs.get("litellm_params")).get("metadata")) + """The request's metadata as the proxy saw it: the caller's bucket with the proxy-internal one on top.""" + litellm_params: Final = as_str_mapping(kwargs.get("litellm_params")) + return _merged(litellm_params.get("metadata"), litellm_params.get("litellm_metadata")) + + +def _merged(metadata: object, litellm_metadata: object) -> Mapping[str, object]: + return {**as_str_mapping(metadata), **as_str_mapping(litellm_metadata)} + + +def _routed_by(metadata: Mapping[str, object], router_name: str) -> bool: + """True when the Router stamped this request as routed by this ORACLE router. + + The Router writes or clears ``routing_decision`` on every routing attempt, so a caller cannot make a + request to some other model look like one of this router's by putting ORACLE keys in its metadata. + """ + decision: Final = as_str_mapping(metadata.get("routing_decision")) + return decision.get("router_type") == "oracle" and decision.get("router_model_name") == router_name + + +def _owner(metadata: Mapping[str, object]) -> str | None: + """The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy.""" + value: Final = metadata.get("user_api_key_hash") + return str(value) if value else None def _score(metadata: Mapping[str, object]) -> float | None: @@ -54,10 +77,10 @@ class OracleRouterPostCallHook(CustomLogger): request_headers: dict[str, str] | None = None, # mutable-ok: CustomLogger's hook signature litellm_call_info: dict[str, object] | None = None, # mutable-ok: CustomLogger's hook signature ) -> dict[str, str] | None: # mutable-ok: CustomLogger's hook signature - buckets: Final = (as_str_mapping(data.get("litellm_metadata")), as_str_mapping(data.get("metadata"))) - chosen: Final = next( - (bucket[CHOSEN_MODEL_METADATA_KEY] for bucket in buckets if CHOSEN_MODEL_METADATA_KEY in bucket), None - ) + metadata: Final = _merged(data.get("metadata"), data.get("litellm_metadata")) + if not _routed_by(metadata, self.oracle_router.router_name): + return None + chosen: Final = metadata.get(CHOSEN_MODEL_METADATA_KEY) return {RESPONSE_HEADER: str(chosen)} if chosen else None async def async_log_success_event( @@ -73,9 +96,12 @@ class OracleRouterPostCallHook(CustomLogger): def _record(self, kwargs: Mapping[str, object], response_obj: object, succeeded: bool) -> None: try: metadata: Final = _request_metadata(kwargs) + if not _routed_by(metadata, self.oracle_router.router_name): + return program_id: Final = metadata.get(PROGRAM_ID_METADATA_KEY) if not isinstance(program_id, str): return + owner: Final = _owner(metadata) if succeeded: cost: Final = kwargs.get("response_cost") self.oracle_router.observe_request( @@ -83,8 +109,9 @@ class OracleRouterPostCallHook(CustomLogger): cost=float(cost) if isinstance(cost, (int, float)) else 0.0, response_text=response_text(response_obj), messages=kwargs.get("messages"), + owner=owner, ) if metadata.get(PROGRAM_DONE_KEY): - self.oracle_router.complete(program_id, score=_score(metadata)) + self.oracle_router.complete(program_id, score=_score(metadata), owner=owner) except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes verbose_router_logger.exception("OracleRouterPostCallHook: failed to record request: %s", error) diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index 4cb223f19cc..084056afacd 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -14,7 +14,7 @@ from collections import deque from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, replace from types import MappingProxyType -from typing import Final +from typing import Final, NamedTuple from pydantic import TypeAdapter, ValidationError @@ -54,6 +54,13 @@ def as_str_mapping(value: object) -> Mapping[str, object]: return _EMPTY +class ProgramKey(NamedTuple): + """A program is private to the API key that started it: the same id under another key is another program.""" + + owner: str | None + program_id: str + + @dataclass(frozen=True, slots=True) class ProgramBinding: """One row of the lookup table: program id to model, plus what the proxy has seen of the program.""" @@ -127,10 +134,29 @@ def _transcript(messages: object, fallback: Sequence[AllMessageValues]) -> Seque def _api_key_hash(request_kwargs: Mapping[str, object]) -> str | None: + """The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy.""" value: Final = _metadata(request_kwargs).get("user_api_key_hash") return str(value) if value else None +def _stamp_internal( + request_kwargs: dict[str, object], # mutable-ok: pre-routing contract + key: str, + value: object | None, +) -> None: + """Write a router-internal metadata key where the proxy keeps its own state. + + Any copy the caller supplied is dropped from both buckets first, so the post-call hook only ever reads + what this router wrote; None clears the key. + """ + for bucket in (request_kwargs.get("metadata"), request_kwargs.get("litellm_metadata")): + if isinstance(bucket, dict): + bucket.pop(key, None) # pyright: ignore[reportUnknownMemberType] # caller-supplied dict + if value is not None: + _, internal = get_or_create_metadata_bucket(request_kwargs) # pyright: ignore[reportUnknownVariableType] # bare dict upstream + internal[key] = value + + class OracleRouter: """One instance per ``auto_router/oracle_router`` deployment.""" @@ -150,8 +176,8 @@ class OracleRouter: self._cause: Final[RoutingDecisionCause] = ( "bandit" if config.decision_maker.type in _LEARNING_DECISION_MAKERS else "classifier_plugin" ) - self._bindings: Final[dict[str, ProgramBinding]] = {} # mutable-ok: the lookup table - self._binding_in_flight: Final[dict[str, asyncio.Event]] = {} # mutable-ok: programs being bound right now + self._bindings: Final[dict[ProgramKey, ProgramBinding]] = {} # mutable-ok: the lookup table + self._binding_in_flight: Final[dict[ProgramKey, asyncio.Event]] = {} # mutable-ok: being bound right now self._tasks: Final[set[asyncio.Task[float]]] = set() # mutable-ok: in-flight verifications self._history: Final[deque[FeedbackRecord]] = deque(maxlen=FEEDBACK_HISTORY_SIZE) # mutable-ok: bounded log self._update_lock: Final = asyncio.Lock() @@ -168,53 +194,60 @@ class OracleRouter: def context_for(self, program_id: str, prompt: str) -> ProgramContext: return ProgramContext(program_id=program_id, prompt=prompt, request_type=classify_prompt(prompt)) - def binding(self, program_id: str) -> ProgramBinding | None: - return self._bindings.get(program_id) + def binding(self, program_id: str, owner: str | None = None) -> ProgramBinding | None: + """The program's row as seen by ``owner``, the key that started it (``user_api_key_hash``).""" + return self._bindings.get(ProgramKey(owner, program_id)) + + def bindings_named(self, program_id: str) -> tuple[ProgramBinding, ...]: + """Every key's program with this id, oldest first: an admin may complete another key's program.""" + matching: Final = (binding for key, binding in self._bindings.items() if key.program_id == program_id) + return tuple(sorted(matching, key=lambda binding: binding.bound_at)) def _evict_expired(self, now: float) -> None: deadline: Final = now - self.config.program_ttl_seconds - expired: Final = tuple( - program_id for program_id, binding in self._bindings.items() if binding.bound_at < deadline - ) - for program_id in expired: - self._bindings.pop(program_id, None) + expired: Final = tuple(key for key, binding in self._bindings.items() if binding.bound_at < deadline) + for key in expired: + self._bindings.pop(key, None) overflow: Final = len(self._bindings) + 1 - self.config.max_programs # +1: room for the program being bound - oldest: Final = tuple(sorted(self._bindings, key=lambda pid: self._bindings[pid].bound_at))[: max(overflow, 0)] - for program_id in oldest: - self._bindings.pop(program_id, None) + oldest: Final = tuple(sorted(self._bindings, key=lambda key: self._bindings[key].bound_at))[: max(overflow, 0)] + for key in oldest: + self._bindings.pop(key, None) self.programs_evicted += len(expired) + len(oldest) async def bind(self, program_id: str, prompt: str, api_key_hash: str | None) -> ProgramBinding: """Bind a new program, or return its existing row. - Concurrent first requests of one program (a harness that fans out) wait for the decision in - flight instead of each asking the decision maker, so a program is bound exactly once. + A program is private to the key that started it: the same id under another key starts that key's + own program instead of riding on, or completing, someone else's. Concurrent first requests of one + program (a harness that fans out) wait for the decision in flight instead of each asking the + decision maker, so a program is bound exactly once. """ - in_flight = self._binding_in_flight.get(program_id) # rebind-ok: re-read after each wait + key: Final = ProgramKey(api_key_hash, program_id) + in_flight = self._binding_in_flight.get(key) # rebind-ok: re-read after each wait while in_flight is not None: await in_flight.wait() - in_flight = self._binding_in_flight.get(program_id) - existing: Final = self._bindings.get(program_id) + in_flight = self._binding_in_flight.get(key) + existing: Final = self._bindings.get(key) if existing is not None: return existing event: Final = asyncio.Event() - self._binding_in_flight[program_id] = event + self._binding_in_flight[key] = event try: - return await self._bind(program_id, prompt, api_key_hash) + return await self._bind(key, prompt) finally: - self._binding_in_flight.pop(program_id, None) + self._binding_in_flight.pop(key, None) event.set() - async def _bind(self, program_id: str, prompt: str, api_key_hash: str | None) -> ProgramBinding: + async def _bind(self, key: ProgramKey, prompt: str) -> ProgramBinding: now: Final = self._clock() if len(self._bindings) >= min(PROGRAM_SWEEP_THRESHOLD, self.config.max_programs): self._evict_expired(now) - context: Final = self.context_for(program_id, prompt) + context: Final = self.context_for(key.program_id, prompt) model: Final = await self.decision_maker.select(context) binding: Final = ProgramBinding( - program_id=program_id, model=model, context=context, api_key_hash=api_key_hash, bound_at=now + program_id=key.program_id, model=model, context=context, api_key_hash=key.owner, bound_at=now ) - self._bindings[program_id] = binding + self._bindings[key] = binding self.programs_bound += 1 return binding @@ -230,9 +263,8 @@ class OracleRouter: program_id: Final = resolve_program_id(request_kwargs, self.config.program_id_key) prompt: Final = first_user_text(messages) chosen_model, request_type = await self._decide(program_id, prompt, _api_key_hash(request_kwargs)) - _, bucket = get_or_create_metadata_bucket(request_kwargs) # pyright: ignore[reportUnknownVariableType] # bare dict upstream - bucket[CHOSEN_MODEL_METADATA_KEY] = chosen_model - bucket[PROGRAM_ID_METADATA_KEY] = program_id + _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) + _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) verbose_router_logger.debug( "OracleRouter[%s]: program=%s request_type=%s -> %s", self.router_name, @@ -260,13 +292,16 @@ class OracleRouter: binding: Final = await self.bind(program_id, prompt, api_key_hash) return binding.model, binding.context.request_type - def observe_request(self, program_id: str, cost: float, response_text: str, messages: object = None) -> None: + def observe_request( + self, program_id: str, cost: float, response_text: str, messages: object = None, owner: str | None = None + ) -> None: """Account one finished LLM request of the program: LiteLLM's spend, the latest answer and transcript.""" - binding: Final = self._bindings.get(program_id) + key: Final = ProgramKey(owner, program_id) + binding: Final = self._bindings.get(key) if binding is None: return transcript: Final = _transcript(messages, binding.last_messages) - self._bindings[program_id] = replace( + self._bindings[key] = replace( binding, requests=binding.requests + 1, cost=binding.cost + max(cost, 0.0), @@ -280,9 +315,10 @@ class OracleRouter: score: float | None = None, cost: float | None = None, payload: Mapping[str, object] = _EMPTY, + owner: str | None = None, ) -> asyncio.Task[float] | None: """Release the program now and verify it in the background; the task resolves to the verified score.""" - binding: Final = self._bindings.pop(program_id, None) + binding: Final = self._bindings.pop(ProgramKey(owner, program_id), None) if binding is None: return None outcome: Final = ProgramOutcome( diff --git a/litellm/types/router.py b/litellm/types/router.py index e4dee4e0085..5a7d61b155b 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1258,7 +1258,8 @@ class OracleDecisionMakerConfig(BaseModel): are that bandit's quality/cost weights. ``fixed`` always picks ``model``. ``pre_routing`` lets the strategy router named by ``router`` (a complexity, quality, adaptive or semantic router in the same model_list) propose the model once per program. ``custom`` imports ``path`` (``package.module:Factory``) and calls it - with the model tuple. + with the model tuple; it is honoured only for deployments in the proxy config file, never for models + written through the API. """ type: OracleDecisionMakerType = "thompson" @@ -1283,7 +1284,8 @@ class OracleVerifierConfig(BaseModel): ``guardrail`` applies the configured LiteLLM guardrail ``guardrail`` (for example an ``llm_as_a_judge`` guardrail) to the program's final turn with the transcript as context. ``reported`` trusts ``metadata.program_score`` or the feedback endpoint. ``custom`` imports ``path`` and calls it with no - arguments. + arguments; it is honoured only for deployments in the proxy config file, never for models written + through the API. """ type: OracleVerifierType = "reported" diff --git a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py index a3d73861634..7d0055dfebb 100644 --- a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py @@ -62,7 +62,7 @@ async def _bind(oracle: OracleRouter, program_id: str, api_key_hash: str | None @pytest.mark.asyncio async def test_feedback_completes_the_program_and_the_router_learns(oracle): await _bind(oracle, "task-1") - oracle.observe_request("task-1", cost=0.2, response_text="done") + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash") with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): response = await submit_oracle_router_feedback( OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER @@ -73,22 +73,35 @@ async def test_feedback_completes_the_program_and_the_router_learns(oracle): "model": "smart", "pending_verifications": 1, } - assert oracle.binding("task-1") is None + assert oracle.binding("task-1", "owner-hash") is None await oracle.drain() assert oracle.maker.updates == [("smart", 1.0)] @pytest.mark.asyncio -async def test_feedback_from_another_key_is_forbidden_but_an_admin_may_complete(oracle): +async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it(oracle): await _bind(oracle, "task-1") with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): - with pytest.raises(HTTPException) as denied: + with pytest.raises(HTTPException) as unseen: await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), STRANGER) - assert denied.value.status_code == 403 and oracle.binding("task-1") is not None + assert unseen.value.status_code == 404 and oracle.binding("task-1", "owner-hash") is not None response = await submit_oracle_router_feedback( OracleRouterFeedbackRequest(program_id="task-1", score=0.0), ADMIN ) - assert response.program_id == "task-1" and oracle.binding("task-1") is None + assert response.program_id == "task-1" and oracle.binding("task-1", "owner-hash") is None + await oracle.drain() + + +@pytest.mark.asyncio +async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across_keys(oracle): + await _bind(oracle, "task-1", "owner-hash") + await _bind(oracle, "task-1", "other-hash") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=0.5), STRANGER + ) + assert response.program_id == "task-1" + assert oracle.binding("task-1", "other-hash") is None and oracle.binding("task-1", "owner-hash") is not None await oracle.drain() diff --git a/tests/unit/router_strategy/oracle_router/test_hooks.py b/tests/unit/router_strategy/oracle_router/test_hooks.py index ebc329a8244..d53b54f5e48 100644 --- a/tests/unit/router_strategy/oracle_router/test_hooks.py +++ b/tests/unit/router_strategy/oracle_router/test_hooks.py @@ -34,11 +34,17 @@ class _Maker: return {} -def _kwargs(program_id="t1", cost=0.01, **metadata): +ROUTED = {"router_model_name": "oracle", "router_type": "oracle"} # what the Router stamps for this router + + +def _kwargs(program_id="t1", cost=0.01, routing_decision=ROUTED, **metadata): + stamped = {PROGRAM_ID_METADATA_KEY: program_id, **metadata} + if routing_decision is not None: + stamped["routing_decision"] = routing_decision return { "model": "openai/gpt-4o", "messages": [{"role": "user", "content": "task"}], - "litellm_params": {"metadata": {PROGRAM_ID_METADATA_KEY: program_id, **metadata}}, + "litellm_params": {"metadata": stamped}, "response_cost": cost, } @@ -113,18 +119,51 @@ async def test_events_for_unknown_or_untagged_programs_are_ignored(bound): assert router.binding("t1").requests == 0 and router.binding("other") is None +@pytest.mark.asyncio +async def test_events_from_requests_this_router_did_not_route_are_ignored(bound): + router, maker, hook = bound + unrouted = _kwargs(cost=0.5, program_done=True, program_score="1", routing_decision=None) + await hook.async_log_success_event(unrouted, _response(), None, None) + elsewhere = _kwargs( + cost=0.5, program_done=True, program_score="1", routing_decision={**ROUTED, "router_model_name": "x"} + ) + await hook.async_log_success_event(elsewhere, _response(), None, None) + assert router.binding("t1").requests == 0 and maker.updates == [] + + +@pytest.mark.asyncio +async def test_events_only_reach_the_program_of_the_key_that_sent_them(bound): + router, maker, hook = bound # t1 was started without a key + stranger = _kwargs(cost=0.5, program_done=True, program_score="1", user_api_key_hash="other-key") + await hook.async_log_success_event(stranger, _response(), None, None) + assert router.binding("t1").requests == 0 and maker.updates == [] + await router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "t1", "user_api_key_hash": "other-key"}}, + messages=[{"role": "user", "content": "task"}], + ) + await hook.async_log_success_event(stranger, _response(), None, None) + assert router.binding("t1", "other-key") is None and router.binding("t1").requests == 0 + await router.drain() + assert maker.updates == [("smart", 1.0)] + + @pytest.mark.asyncio async def test_response_header_names_the_bound_model(bound): _, _, hook = bound headers = await hook.async_post_call_response_headers_hook( - {"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart"}}, None, None + {"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart", "routing_decision": ROUTED}}, None, None ) assert headers == {RESPONSE_HEADER: "smart"} - assert await hook.async_post_call_response_headers_hook({"metadata": {}}, None, None) is None + assert ( + await hook.async_post_call_response_headers_hook({"metadata": {"routing_decision": ROUTED}}, None, None) is None + ) + forged = {"metadata": {CHOSEN_MODEL_METADATA_KEY: "smart"}} # a request this router did not route + assert await hook.async_post_call_response_headers_hook(forged, None, None) is None @pytest.mark.asyncio async def test_response_header_reads_the_litellm_metadata_bucket_too(bound): _, _, hook = bound - data = {"metadata": {}, "litellm_metadata": {CHOSEN_MODEL_METADATA_KEY: "fast"}} + data = {"metadata": {}, "litellm_metadata": {CHOSEN_MODEL_METADATA_KEY: "fast", "routing_decision": ROUTED}} assert await hook.async_post_call_response_headers_hook(data, None, None) == {RESPONSE_HEADER: "fast"} diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index 240f93a7fe4..966bd905f79 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -98,13 +98,13 @@ async def test_first_request_binds_and_later_requests_reuse_the_binding(): ) assert response.routing_decision["router_type"] == "oracle" and response.routing_decision["routed_model"] == "smart" - later = {"metadata": {"program_id": "t1"}} + later = {"metadata": {"program_id": "t1", "user_api_key_hash": "key-1"}} again = await router.async_pre_routing_hook( model="oracle", request_kwargs=later, messages=_messages("now run the tests") ) assert again is not None and again.model == "smart" assert maker.selections == 1 - assert router.binding("t1").api_key_hash == "key-1" + assert router.binding("t1", "key-1").api_key_hash == "key-1" other = await router.async_pre_routing_hook( model="oracle", request_kwargs={"metadata": {"program_id": "t2"}}, messages=_messages() @@ -152,7 +152,7 @@ async def test_requests_without_a_program_id_route_statelessly_and_never_learn() router, maker = _router() kwargs = {"metadata": {}} response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) - assert response.model == "smart" and kwargs["metadata"][PROGRAM_ID_METADATA_KEY] is None + assert response.model == "smart" and PROGRAM_ID_METADATA_KEY not in kwargs["metadata"] assert router.stateless_requests == 1 and router.binding("") is None assert router.complete("", score=1.0) is None @@ -238,4 +238,38 @@ async def test_decision_is_stamped_in_the_proxy_internal_bucket_when_one_exists( assert response.model == "smart" assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == "smart" assert CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"] - assert router.binding("t1").api_key_hash == "k" + assert router.binding("t1", "k").api_key_hash == "k" + + +@pytest.mark.asyncio +async def test_the_same_program_id_under_another_key_is_another_program(): + router, maker = _router() + for key in ("key-a", "key-b"): + await router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "t1", "user_api_key_hash": key}}, + messages=_messages(), + ) + assert router.programs_bound == 2 and maker.selections == 2 + assert router.binding("t1") is None # no key: not the same program either + assert [binding.api_key_hash for binding in router.bindings_named("t1")] == ["key-a", "key-b"] + router.complete("t1", score=1.0, owner="key-b") + assert router.binding("t1", "key-a") is not None and router.binding("t1", "key-b") is None + await router.drain() + + +@pytest.mark.asyncio +async def test_pre_routing_hook_replaces_caller_supplied_internal_keys(): + router, _ = _router() + kwargs = { + "metadata": {"program_id": "t1", PROGRAM_ID_METADATA_KEY: "victim", CHOSEN_MODEL_METADATA_KEY: "fast"}, + "litellm_metadata": {PROGRAM_ID_METADATA_KEY: "victim"}, + } + response = await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) + assert PROGRAM_ID_METADATA_KEY not in kwargs["metadata"] and CHOSEN_MODEL_METADATA_KEY not in kwargs["metadata"] + assert kwargs["litellm_metadata"][PROGRAM_ID_METADATA_KEY] == "t1" + assert kwargs["litellm_metadata"][CHOSEN_MODEL_METADATA_KEY] == response.model + stateless = {"metadata": {PROGRAM_ID_METADATA_KEY: "victim", CHOSEN_MODEL_METADATA_KEY: "fast"}} + await router.async_pre_routing_hook(model="oracle", request_kwargs=stateless, messages=_messages()) + assert PROGRAM_ID_METADATA_KEY not in stateless["metadata"] + assert stateless["metadata"][CHOSEN_MODEL_METADATA_KEY] in ("smart", "fast") diff --git a/tests/unit/router_strategy/oracle_router/test_router_dispatch.py b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py index 6e6d026606b..7dc55f063b6 100644 --- a/tests/unit/router_strategy/oracle_router/test_router_dispatch.py +++ b/tests/unit/router_strategy/oracle_router/test_router_dispatch.py @@ -35,6 +35,20 @@ def _oracle_hooks(): return litellm.logging_callback_manager.get_custom_loggers_for_type(OracleRouterPostCallHook) +class _CustomMaker: + def __init__(self, models): + self.models = tuple(models) + + async def select(self, context): + return self.models[-1] + + def update(self, context, model, score): + pass + + def snapshot(self): + return {"type": "custom"} + + def test_prefix_is_classified_as_oracle_and_not_as_a_semantic_router(): router = Router(model_list=[]) params = LiteLLM_Params(model="auto_router/oracle_router") @@ -202,6 +216,37 @@ def test_upserting_an_oracle_deployment_builds_its_router_once_and_rebuilds_it_w assert [hook.oracle_router for hook in _oracle_hooks() if hook.oracle_router in (oracle, rebuilt)] == [rebuilt] +def test_custom_plugins_load_from_the_config_file_but_never_for_models_written_through_the_api(monkeypatch): + import subprocess + + popen_calls = [] + monkeypatch.setattr(subprocess, "Popen", lambda *args, **kwargs: popen_calls.append(args)) + from_config = Router(model_list=_model_list(decision_maker={"type": "custom", "path": f"{__name__}:_CustomMaker"})) + assert _oracle(from_config, "oracle").decision_maker.models == ("smart", "fast") + for config_key in ("decision_maker", "verifier"): + model_list = _model_list(**{config_key: {"type": "custom", "path": "subprocess:Popen"}}) + model_list[0]["model_info"] = {"db_model": True} # what the proxy sets for /model/new and /model/update + model_list[0]["litellm_params"]["oracle_router_config"]["available_models"] = ["/bin/sh", "-c", "id"] + with pytest.raises(ValueError, match="config file"): + Router(model_list=model_list) + router = Router(model_list=_model_list()[1:]) + with pytest.raises(ValueError, match="config file"): + router.upsert_deployment( + deployment=Deployment( + model_name="api-oracle", + litellm_params=LiteLLM_Params( + model="auto_router/oracle_router", + oracle_router_config={ + "available_models": ["smart", "fast"], + "verifier": {"type": "custom", "path": "subprocess:Popen"}, + }, + ), + model_info={"id": "api-oracle-1", "db_model": True}, + ) + ) + assert popen_calls == [] and "api-oracle" not in router.oracle_routers + + def test_strategy_router_dependencies_cover_available_models(): from litellm.router_utils.auto_router_model_naming import strategy_router_dependencies From 3ea3715648c819e560c55567e430271951976b0d Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Sat, 3 Oct 2026 19:40:11 +0000 Subject: [PATCH 05/10] fix(oracle_router): regenerate schema.d.ts; keep program ids out of log lines - ui/litellm-dashboard/src/lib/http/schema.d.ts follows the feedback endpoint's new description (check-ui-api-types). - The pre-routing debug line and the verification-failure log no longer include the caller-supplied program id (it is on /oracle_router/state): CodeQL's py/log-injection evaluation on the PR exceeded its 2 GiB result limit tracing request metadata into these sinks, and a program id with newlines could forge log lines. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- .../router_strategy/oracle_router/oracle_router.py | 12 ++++++++---- ui/litellm-dashboard/src/lib/http/schema.d.ts | 4 ++-- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index 084056afacd..65b22e730d4 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -265,10 +265,10 @@ class OracleRouter: chosen_model, request_type = await self._decide(program_id, prompt, _api_key_hash(request_kwargs)) _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) - verbose_router_logger.debug( - "OracleRouter[%s]: program=%s request_type=%s -> %s", + verbose_router_logger.debug( # caller-supplied program ids stay out of the log + "OracleRouter[%s]: %s request, request_type=%s -> %s", self.router_name, - program_id, + "program" if program_id else "stateless", request_type.value, chosen_model, ) @@ -343,7 +343,11 @@ class OracleRouter: except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop self.verifications_failed += 1 verbose_router_logger.exception( - "OracleRouter[%s]: verification of %s failed: %s", self.router_name, binding.program_id, error + "OracleRouter[%s]: verification of a %s program on %s failed: %s", + self.router_name, + binding.context.request_type.value, + binding.model, + error, ) return float("nan") async with self._update_lock: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index ebb7ffc5840..135530dbc1b 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11393,8 +11393,8 @@ export interface paths { * Submit Oracle Router Feedback * @description Complete a program: its model slot is released now, its verifier runs in the background. * - * The key that started the program (or an admin) may complete it. 404 when no ORACLE router holds a - * program with that id, which also covers programs completed earlier. + * Programs are private to the key that started them; an admin may complete any key's program. 404 when + * the caller holds no program with that id, which also covers programs completed earlier. */ post: operations["submit_oracle_router_feedback_oracle_router_feedback_post"]; delete?: never; From de79b119221614a4f974bc4e348050b95038cb76 Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Sat, 3 Oct 2026 20:34:35 +0000 Subject: [PATCH 06/10] fix(oracle_router): feedback payloads cannot pre-seed the guardrail's status record - The guardrail verifier reads its verdict from the status record the guardrail appends to the request metadata it is handed, and that metadata was the caller's feedback payload spread as-is. A payload carrying `standard_logging_guardrail_information: [0]` made the recorded list unreadable, so a rejected answer under `on_failure: log` scored 1.0. The status key is now dropped from the payload before the guardrail runs, and the payload can no longer rename the program. - Log lines in the router and its hook no longer carry anything derived from the request (program id, prompt classification, chosen model, exception text): `logger.exception` keeps the traceback, and CodeQL's log-injection query no longer has request-fed sinks to trace through the Router. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- litellm/router_strategy/oracle_router/hooks.py | 4 ++-- .../oracle_router/oracle_router.py | 18 ++++-------------- .../router_strategy/oracle_router/verifier.py | 15 ++++++++++++++- .../oracle_router/test_verifier.py | 17 ++++++++++++++--- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py index 8ad8bded760..77479b2d1ef 100644 --- a/litellm/router_strategy/oracle_router/hooks.py +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -113,5 +113,5 @@ class OracleRouterPostCallHook(CustomLogger): ) if metadata.get(PROGRAM_DONE_KEY): self.oracle_router.complete(program_id, score=_score(metadata), owner=owner) - except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes - verbose_router_logger.exception("OracleRouterPostCallHook: failed to record request: %s", error) + except Exception: # noqa: BLE001 # a logging callback must never fail the request it observes + verbose_router_logger.exception("OracleRouterPostCallHook: failed to record a request") diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index 65b22e730d4..eb3aff11667 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -265,12 +265,8 @@ class OracleRouter: chosen_model, request_type = await self._decide(program_id, prompt, _api_key_hash(request_kwargs)) _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) - verbose_router_logger.debug( # caller-supplied program ids stay out of the log - "OracleRouter[%s]: %s request, request_type=%s -> %s", - self.router_name, - "program" if program_id else "stateless", - request_type.value, - chosen_model, + verbose_router_logger.debug( # nothing request-derived is logged; the decision is on the request's metadata + "OracleRouter: routed a %s request", "program" if program_id else "stateless" ) return PreRoutingHookResponse( model=chosen_model, @@ -340,15 +336,9 @@ class OracleRouter: async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float: try: score: Final = clamp_score(await self.verifier.verify(outcome)) - except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop + except Exception: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop self.verifications_failed += 1 - verbose_router_logger.exception( - "OracleRouter[%s]: verification of a %s program on %s failed: %s", - self.router_name, - binding.context.request_type.value, - binding.model, - error, - ) + verbose_router_logger.exception("OracleRouter: a verification failed") return float("nan") async with self._update_lock: self.decision_maker.update(binding.context, binding.model, score) diff --git a/litellm/router_strategy/oracle_router/verifier.py b/litellm/router_strategy/oracle_router/verifier.py index 5419a5000e6..e5c64fcf38b 100644 --- a/litellm/router_strategy/oracle_router/verifier.py +++ b/litellm/router_strategy/oracle_router/verifier.py @@ -82,6 +82,19 @@ def find_guardrail(name: str) -> CustomGuardrail | None: return next((guardrail for guardrail in guardrails if guardrail.guardrail_name == name), None) +def _verifier_metadata(outcome: ProgramOutcome) -> dict[str, object]: # mutable-ok: the guardrail writes into it + """The feedback payload as the guardrail's request metadata. + + The verdict is read from the status record the guardrail writes here, so the caller's payload must not + pre-seed that key (a non-record entry would make the whole list unreadable and hide a rejection), and + it cannot rename the program. + """ + return { + **{key: value for key, value in outcome.payload.items() if key != _STATUS_RECORD_KEY}, + "program_id": outcome.program_id, + } + + def _status_records(bucket: object) -> Sequence[Mapping[str, object]]: """The ``standard_logging_guardrail_information`` records of one metadata bucket, or none.""" try: @@ -163,7 +176,7 @@ class GuardrailVerifier: ) request_data: Final[dict[str, object]] = { # mutable-ok: the guardrail records its status into it "model": outcome.model, - "metadata": {"program_id": outcome.program_id, **outcome.payload}, + "metadata": _verifier_metadata(outcome), } verdict_error: Final = await _apply(guardrail, inputs, request_data) status: Final = recorded_status(request_data, self._name) diff --git a/tests/unit/router_strategy/oracle_router/test_verifier.py b/tests/unit/router_strategy/oracle_router/test_verifier.py index 20931fa8b97..a9219799faa 100644 --- a/tests/unit/router_strategy/oracle_router/test_verifier.py +++ b/tests/unit/router_strategy/oracle_router/test_verifier.py @@ -53,9 +53,9 @@ class _RecordingGuardrail: self.status, self.error, self.name = status, error, name async def apply_guardrail(self, inputs, request_data, input_type, logging_obj=None): - request_data["metadata"]["standard_logging_guardrail_information"] = [ - {"guardrail_name": self.name, "guardrail_status": self.status} - ] + self.seen = dict(request_data["metadata"]) # what the guardrail was handed, before it records anything + records = request_data["metadata"].setdefault("standard_logging_guardrail_information", []) + records.append({"guardrail_name": self.name, "guardrail_status": self.status}) # appended, like the real ones if self.error is not None: raise self.error return inputs @@ -104,6 +104,17 @@ async def test_guardrail_verifier_reads_the_recorded_verdict_before_the_exceptio assert await GuardrailVerifier("flagged", lookup=judge.get).verify(_outcome()) == 0.0 +@pytest.mark.asyncio +async def test_feedback_payload_cannot_pre_seed_the_status_record_or_rename_the_program(): + judge = _RecordingGuardrail("guardrail_intervened", name="judge") # on_failure: log -> no exception + forged = {"standard_logging_guardrail_information": [0], "program_id": "someone-else", "container": "c1"} + outcome = ProgramOutcome( + program_id="p", model="smart", prompt="task", response_text="wrong", cost=0.0, messages=(), payload=forged + ) + assert await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(outcome) == 0.0 + assert judge.seen == {"program_id": "p", "container": "c1"} + + @pytest.mark.asyncio async def test_guardrail_that_could_not_evaluate_is_a_failed_verification_not_a_pass(): judge = { From 439dc0c0dec8a30cb0ea85c492ad5f147de7814d Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Sat, 3 Oct 2026 22:34:07 +0000 Subject: [PATCH 07/10] fix(oracle_router): never verify a program that has no answer yet - Feedback (the endpoint, or program_done on a failed request) that arrives before any response of the program was observed no longer completes it: the completion is deferred on the binding and applied by the next observed response. Before, a harness could complete a program whose answer was still streaming; llm_as_a_judge skipped the empty answer without a verdict and the program scored 1.0. The feedback reply says `deferred: true`. - The guardrail verifier fails the verification of a program whose final response has no text instead of reading a judge's skip as a pass. - schema.d.ts regenerated for the new response field. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- .../oracle_router_endpoints.py | 5 +++- .../router_strategy/oracle_router/README.md | 9 +++++-- .../oracle_router/oracle_router.py | 27 +++++++++++++++++-- .../router_strategy/oracle_router/verifier.py | 3 +++ .../oracle_router_endpoints.py | 4 +++ .../test_oracle_router_endpoints.py | 18 +++++++++++++ .../oracle_router/test_hooks.py | 5 +++- .../oracle_router/test_oracle_router.py | 20 ++++++++++++++ .../oracle_router/test_verifier.py | 8 ++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 ++++++- 10 files changed, 102 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/oracle_router_endpoints.py b/litellm/proxy/management_endpoints/oracle_router_endpoints.py index a88bab1fe6e..63105933658 100644 --- a/litellm/proxy/management_endpoints/oracle_router_endpoints.py +++ b/litellm/proxy/management_endpoints/oracle_router_endpoints.py @@ -70,7 +70,9 @@ async def submit_oracle_router_feedback( """Complete a program: its model slot is released now, its verifier runs in the background. Programs are private to the key that started them; an admin may complete any key's program. 404 when - the caller holds no program with that id, which also covers programs completed earlier. + the caller holds no program with that id, which also covers programs completed earlier. A program none + of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed + response completes it with this feedback. """ routers: Final = _oracle_routers() if not routers: @@ -87,6 +89,7 @@ async def submit_oracle_router_feedback( router_name=holder.router_name, model=binding.model, pending_verifications=holder.pending_verifications, + deferred=binding.requests == 0, ) diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index 9d05c3ae81e..b458010e7c3 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -48,8 +48,13 @@ never reaches the decision maker. Either mark the last request with `metadata.program_done: true` (plus `program_score` for the reported verifier), or call `POST /oracle_router/feedback {"program_id": ..., "score": ..., "cost": ...}` from -the harness once the task is graded. `GET /oracle_router/state` (admin) shows bindings, decision-maker -state and recent feedback. The response header `x-litellm-oracle-router-model` names the bound model. +the harness once the task is graded. Feedback that arrives before any response of the program was +observed (it raced the last response's logging, or every request failed) is deferred: the reply says +`deferred: true` and the next observed response completes the program with it, so nothing is ever +verified or learned without an answer. A guardrail verifier also fails the verification of a program +whose final response has no text, rather than reading a judge's skip as a pass. `GET /oracle_router/state` +(admin) shows bindings, decision-maker state and recent feedback. The response header +`x-litellm-oracle-router-model` names the bound model. ## Security diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index eb3aff11667..98bbc6e27ba 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -61,6 +61,15 @@ class ProgramKey(NamedTuple): program_id: str +@dataclass(frozen=True, slots=True) +class DeferredCompletion: + """A completion that arrived before any response of the program was observed; the next one applies it.""" + + score: float | None + cost: float | None + payload: Mapping[str, object] + + @dataclass(frozen=True, slots=True) class ProgramBinding: """One row of the lookup table: program id to model, plus what the proxy has seen of the program.""" @@ -74,6 +83,7 @@ class ProgramBinding: cost: float = 0.0 last_response_text: str = "" last_messages: Sequence[AllMessageValues] = () + deferred: DeferredCompletion | None = None @dataclass(frozen=True, slots=True) @@ -304,6 +314,9 @@ class OracleRouter: last_response_text=response_text or binding.last_response_text, last_messages=transcript, ) + deferred: Final = binding.deferred + if deferred is not None: # the harness reported before this response was observed + self.complete(program_id, score=deferred.score, cost=deferred.cost, payload=deferred.payload, owner=owner) def complete( self, @@ -313,10 +326,20 @@ class OracleRouter: payload: Mapping[str, object] = _EMPTY, owner: str | None = None, ) -> asyncio.Task[float] | None: - """Release the program now and verify it in the background; the task resolves to the verified score.""" - binding: Final = self._bindings.pop(ProgramKey(owner, program_id), None) + """Release the program now and verify it in the background; the task resolves to the verified score. + + A program none of whose requests has been observed yet (feedback that raced the last response's + logging, or a program whose only request failed) keeps its slot and is completed, with these + arguments, by its next observed response: there is no answer to verify yet, so nothing is learned. + """ + key: Final = ProgramKey(owner, program_id) + binding: Final = self._bindings.get(key) if binding is None: return None + if binding.requests == 0: + self._bindings[key] = replace(binding, deferred=DeferredCompletion(score=score, cost=cost, payload=payload)) + return None + del self._bindings[key] outcome: Final = ProgramOutcome( program_id=program_id, model=binding.model, diff --git a/litellm/router_strategy/oracle_router/verifier.py b/litellm/router_strategy/oracle_router/verifier.py index e5c64fcf38b..5b202d90701 100644 --- a/litellm/router_strategy/oracle_router/verifier.py +++ b/litellm/router_strategy/oracle_router/verifier.py @@ -170,6 +170,9 @@ class GuardrailVerifier: guardrail: Final = self._lookup(self._name) if guardrail is None: raise LookupError(f"guardrail {self._name!r} is not initialized on this proxy") + if not outcome.response_text.strip(): + # A judge skips an empty answer without recording a verdict; that must not read as a pass. + raise RuntimeError(f"guardrail {self._name!r}: the program has no response text to verify") transcript: Final = list(outcome.messages) or [{"role": "user", "content": outcome.prompt}] inputs: Final = GenericGuardrailAPIInputs( texts=[outcome.response_text], model=outcome.model, structured_messages=transcript diff --git a/litellm/types/management_endpoints/oracle_router_endpoints.py b/litellm/types/management_endpoints/oracle_router_endpoints.py index cb5f63faf35..e400ee36100 100644 --- a/litellm/types/management_endpoints/oracle_router_endpoints.py +++ b/litellm/types/management_endpoints/oracle_router_endpoints.py @@ -21,6 +21,10 @@ class OracleRouterFeedbackResponse(BaseModel): router_name: str model: str pending_verifications: int + deferred: bool = Field( + default=False, + description="No response of the program has been observed yet; its next one completes it with this feedback", + ) class OracleRouterStateResponse(BaseModel): diff --git a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py index 7d0055dfebb..3c02f4ae8db 100644 --- a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py @@ -72,6 +72,7 @@ async def test_feedback_completes_the_program_and_the_router_learns(oracle): "router_name": "oracle", "model": "smart", "pending_verifications": 1, + "deferred": False, } assert oracle.binding("task-1", "owner-hash") is None await oracle.drain() @@ -81,6 +82,7 @@ async def test_feedback_completes_the_program_and_the_router_learns(oracle): @pytest.mark.asyncio async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it(oracle): await _bind(oracle, "task-1") + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash") with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): with pytest.raises(HTTPException) as unseen: await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), STRANGER) @@ -96,6 +98,7 @@ async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across_keys(oracle): await _bind(oracle, "task-1", "owner-hash") await _bind(oracle, "task-1", "other-hash") + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="other-hash") with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): response = await submit_oracle_router_feedback( OracleRouterFeedbackRequest(program_id="task-1", score=0.5), STRANGER @@ -105,6 +108,21 @@ async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across await oracle.drain() +@pytest.mark.asyncio +async def test_feedback_before_any_response_is_deferred_until_one_is_observed(oracle): + await _bind(oracle, "task-1") # bound, but its first response has not been observed yet + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER + ) + assert response.deferred is True and response.pending_verifications == 0 + assert oracle.binding("task-1", "owner-hash") is not None and oracle.maker.updates == [] + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash") + assert oracle.binding("task-1", "owner-hash") is None + await oracle.drain() + assert oracle.maker.updates == [("smart", 1.0)] + + @pytest.mark.asyncio async def test_feedback_for_an_unknown_program_or_without_routers_is_404(oracle): with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): diff --git a/tests/unit/router_strategy/oracle_router/test_hooks.py b/tests/unit/router_strategy/oracle_router/test_hooks.py index d53b54f5e48..82e9710d755 100644 --- a/tests/unit/router_strategy/oracle_router/test_hooks.py +++ b/tests/unit/router_strategy/oracle_router/test_hooks.py @@ -100,11 +100,14 @@ async def test_a_non_finite_program_score_is_treated_as_no_score(bound): @pytest.mark.asyncio -async def test_failure_event_does_not_add_spend_but_still_honours_program_done(bound): +async def test_failure_event_adds_no_spend_and_program_done_on_it_waits_for_a_response(bound): router, maker, hook = bound await hook.async_log_failure_event(_kwargs(cost=0.5), None, None, None) assert router.binding("t1").cost == 0.0 await hook.async_log_failure_event(_kwargs(program_done=True), None, None, None) + assert router.binding("t1") is not None and maker.updates == [] # no answer to verify yet: deferred + await hook.async_log_success_event(_kwargs(cost=0.1), _response("late answer"), None, None) + assert router.binding("t1") is None await router.drain() assert maker.updates == [("smart", 0.0)] diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index 966bd905f79..2b8608429a5 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -142,6 +142,7 @@ async def test_a_non_finite_verifier_score_is_a_failed_verification(): await router.async_pre_routing_hook( model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() ) + router.observe_request("t", cost=0.1, response_text="answer") result = await router.complete("t") assert result != result and router.verifications_failed == 1 and maker.updates == [] assert router.binding("t") is None # the slot was released all the same @@ -166,6 +167,7 @@ async def test_complete_releases_the_slot_now_and_learns_in_verification_order() ) router.observe_request("t1", cost=0.4, response_text="patch") router.observe_request("t1", cost=0.2, response_text="tests pass") + router.observe_request("t2", cost=0.1, response_text="wrong") t1 = router.complete("t1", score=1.0) t2 = router.complete("t2", score=0.0) assert router.binding("t1") is None and router.binding("t2") is None @@ -210,6 +212,7 @@ async def test_failed_verification_is_counted_and_does_not_update(): await router.async_pre_routing_hook( model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() ) + router.observe_request("t", cost=0.1, response_text="answer") result = await router.complete("t") assert result != result and router.verifications_failed == 1 and maker.updates == [] @@ -253,11 +256,28 @@ async def test_the_same_program_id_under_another_key_is_another_program(): assert router.programs_bound == 2 and maker.selections == 2 assert router.binding("t1") is None # no key: not the same program either assert [binding.api_key_hash for binding in router.bindings_named("t1")] == ["key-a", "key-b"] + router.observe_request("t1", cost=0.1, response_text="answer", owner="key-b") router.complete("t1", score=1.0, owner="key-b") assert router.binding("t1", "key-a") is not None and router.binding("t1", "key-b") is None await router.drain() +@pytest.mark.asyncio +async def test_completion_before_any_observed_response_is_deferred_to_the_next_one(): + router, maker = _router() + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + assert router.complete("t", score=1.0, cost=2.0, payload={"run": "r1"}) is None # nothing answered yet + binding = router.binding("t") + assert binding is not None and binding.deferred is not None and router.programs_completed == 0 + assert maker.updates == [] + router.observe_request("t", cost=0.3, response_text="answer") # the response the feedback raced + assert router.binding("t") is None and router.programs_completed == 1 + await router.drain() + assert maker.updates == [("t", "smart", 1.0)] + + @pytest.mark.asyncio async def test_pre_routing_hook_replaces_caller_supplied_internal_keys(): router, _ = _router() diff --git a/tests/unit/router_strategy/oracle_router/test_verifier.py b/tests/unit/router_strategy/oracle_router/test_verifier.py index a9219799faa..66164f6b914 100644 --- a/tests/unit/router_strategy/oracle_router/test_verifier.py +++ b/tests/unit/router_strategy/oracle_router/test_verifier.py @@ -104,6 +104,14 @@ async def test_guardrail_verifier_reads_the_recorded_verdict_before_the_exceptio assert await GuardrailVerifier("flagged", lookup=judge.get).verify(_outcome()) == 0.0 +@pytest.mark.asyncio +async def test_a_program_without_response_text_is_a_failed_verification_not_a_pass(): + judge = _RecordingGuardrail("success", name="judge") # would skip an empty answer without a record + with pytest.raises(RuntimeError, match="no response text"): + await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(_outcome(answer=" ")) + assert not hasattr(judge, "seen") # the guardrail was never invoked + + @pytest.mark.asyncio async def test_feedback_payload_cannot_pre_seed_the_status_record_or_rename_the_program(): judge = _RecordingGuardrail("guardrail_intervened", name="judge") # on_failure: log -> no exception diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 01230dd3137..18b19bb8925 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11450,7 +11450,9 @@ export interface paths { * @description Complete a program: its model slot is released now, its verifier runs in the background. * * Programs are private to the key that started them; an admin may complete any key's program. 404 when - * the caller holds no program with that id, which also covers programs completed earlier. + * the caller holds no program with that id, which also covers programs completed earlier. A program none + * of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed + * response completes it with this feedback. */ post: operations["submit_oracle_router_feedback_oracle_router_feedback_post"]; delete?: never; @@ -39415,6 +39417,12 @@ export interface components { }; /** OracleRouterFeedbackResponse */ OracleRouterFeedbackResponse: { + /** + * Deferred + * @description No response of the program has been observed yet; its next one completes it with this feedback + * @default false + */ + deferred: boolean; /** Model */ model: string; /** Pending Verifications */ From 0d6e8dfdd328faea8ea90a30676d547bc51e3215 Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Sat, 3 Oct 2026 23:08:57 +0000 Subject: [PATCH 08/10] fix(oracle_router): write nothing to the process log; keep failures on the state endpoint CodeQL's pull-request analysis is diff-informed: it only relates sources and sinks that lie in the PR's changed lines. When a PR adds both a taint source (here the feedback endpoint's request body) and a log-call sink, the data flow library falls back to computing every path in the codebase, and on LiteLLM py/log-injection then exceeds the 2 GiB relation limit. Reproduced with CodeQL 2.27.1 and the action's diff-range extension pack: the full diff fails after 14 minutes, every subset without that combination evaluates in under 20 seconds, and the tree with no log call on any added line evaluates in 15 seconds. So the router, its hook and its Router init log nothing. The last failure of a background step (verification, the post-call hook) is kept as `last_failure` on the router and shown by GET /oracle_router/state, next to the counters it already carried. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- litellm/router.py | 7 ------- .../router_strategy/oracle_router/README.md | 4 +++- .../router_strategy/oracle_router/hooks.py | 8 ++++---- .../oracle_router/oracle_router.py | 19 +++++++++++++------ .../oracle_router/test_oracle_router.py | 2 ++ 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 06fddf79dec..922c7c41031 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9551,13 +9551,6 @@ class Router: strategy=oracle_router, strategy_label="Oracle-router", ) - verbose_router_logger.info( - "OracleRouter[%s] initialized with %d models, decision_maker=%s, verifier=%s", - deployment.model_name, - len(config.available_models), - config.decision_maker.type, - config.verifier.type, - ) def _is_quality_router_deployment(self, litellm_params: LiteLLM_Params) -> bool: """ diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index b458010e7c3..140e9dd67fb 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -72,4 +72,6 @@ whatever the caller's role) with a `custom` type is rejected when the router is ## Limits State is in memory per proxy process; it is not persisted or shared across replicas. The paper's -dispatch scheduler (DISC) is not part of this strategy. +dispatch scheduler (DISC) is not part of this strategy. The router and its hook write nothing to the +process log: `GET /oracle_router/state` carries the counters and the last background failure +(`last_failure`) instead. diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py index 77479b2d1ef..6165f6293e3 100644 --- a/litellm/router_strategy/oracle_router/hooks.py +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -3,14 +3,14 @@ After every completion it adds LiteLLM's computed spend and the latest answer to the program's row, and when the request carried ``metadata.program_done`` it completes the program: the slot is released at once and the verifier runs in the background. Only requests the Router stamped as routed by this ORACLE router -are read, and only against the program of the key that sent them. Nothing here can fail a request. +are read, and only against the program of the key that sent them. Nothing here can fail a request, and +nothing here logs: a failure is kept on the router for ``/oracle_router/state``. """ import math from collections.abc import Mapping from typing import TYPE_CHECKING, Final -from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger from litellm.router_strategy.oracle_router.config import ( CHOSEN_MODEL_METADATA_KEY, @@ -113,5 +113,5 @@ class OracleRouterPostCallHook(CustomLogger): ) if metadata.get(PROGRAM_DONE_KEY): self.oracle_router.complete(program_id, score=_score(metadata), owner=owner) - except Exception: # noqa: BLE001 # a logging callback must never fail the request it observes - verbose_router_logger.exception("OracleRouterPostCallHook: failed to record a request") + except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes + self.oracle_router.note_failure("post-call hook", error) diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index 98bbc6e27ba..0be99e0f1c3 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -18,7 +18,6 @@ from typing import Final, NamedTuple from pydantic import TypeAdapter, ValidationError -from litellm._logging import verbose_router_logger from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict ) @@ -196,11 +195,21 @@ class OracleRouter: self.programs_evicted: int = 0 self.stateless_requests: int = 0 self.verifications_failed: int = 0 + self.last_failure: str | None = None @property def models(self) -> tuple[str, ...]: return tuple(self.config.available_models) + def note_failure(self, step: str, error: Exception) -> None: + """Keep the last failure of a background step for ``/oracle_router/state``. + + The router and its hook write nothing to the process log: every request input reaches them, and a + log call on that path makes CodeQL's log-injection analysis fall back to a whole-codebase pass that + no longer fits its limits on this codebase. The admin state endpoint carries the failure instead. + """ + self.last_failure = f"{step}: {type(error).__name__}: {error}" + def context_for(self, program_id: str, prompt: str) -> ProgramContext: return ProgramContext(program_id=program_id, prompt=prompt, request_type=classify_prompt(prompt)) @@ -275,9 +284,6 @@ class OracleRouter: chosen_model, request_type = await self._decide(program_id, prompt, _api_key_hash(request_kwargs)) _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) - verbose_router_logger.debug( # nothing request-derived is logged; the decision is on the request's metadata - "OracleRouter: routed a %s request", "program" if program_id else "stateless" - ) return PreRoutingHookResponse( model=chosen_model, messages=messages, @@ -359,9 +365,9 @@ class OracleRouter: async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float: try: score: Final = clamp_score(await self.verifier.verify(outcome)) - except Exception: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop + except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop self.verifications_failed += 1 - verbose_router_logger.exception("OracleRouter: a verification failed") + self.note_failure("verification", error) return float("nan") async with self._update_lock: self.decision_maker.update(binding.context, binding.model, score) @@ -410,6 +416,7 @@ class OracleRouter: "stateless_requests": self.stateless_requests, "pending_verifications": self.pending_verifications, "verifications_failed": self.verifications_failed, + "last_failure": self.last_failure, "recent_accuracy": self._accuracy_by_model(), "recent_feedback": [ { diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index 2b8608429a5..f5651cc8b48 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -215,6 +215,8 @@ async def test_failed_verification_is_counted_and_does_not_update(): router.observe_request("t", cost=0.1, response_text="answer") result = await router.complete("t") assert result != result and router.verifications_failed == 1 and maker.updates == [] + snapshot = await router.get_state_snapshot() + assert snapshot["last_failure"] == "verification: RuntimeError: sandbox down" # kept for the state endpoint, not logged @pytest.mark.asyncio From 5e9c9e2e8d4a5d149a6aa29e39fdf5b347a48256 Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Wed, 4 Nov 2026 23:18:52 +0000 Subject: [PATCH 09/10] fix(oracle_router): bound background verification and keep prompt privacy through pre_routing Completing a program released its binding before the verification task started, so max_programs never limited pending verifications and a verifier had no deadline. At most max_pending_verifications verifications run per router and each gets verification_timeout_seconds; a full backlog refuses the completion, keeps the program bound for a retry, answers 503 on the feedback endpoint and counts verifications_rejected on the state endpoint. The pre_routing decision maker called the wrapped strategy with empty metadata, dropping the program's turn_off_message_logging opt-out. The opt-out is kept on the program context and forwarded to the strategy, both in the metadata and through the inherited privacy context. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- .../oracle_router_endpoints.py | 8 ++- .../router_strategy/oracle_router/README.md | 11 ++++ .../router_strategy/oracle_router/decision.py | 13 ++-- .../oracle_router/oracle_router.py | 49 +++++++++++---- litellm/types/router.py | 2 + .../test_oracle_router_endpoints.py | 23 +++++++ .../oracle_router/test_decision.py | 21 +++++++ .../oracle_router/test_oracle_router.py | 60 ++++++++++++++++++- 8 files changed, 168 insertions(+), 19 deletions(-) diff --git a/litellm/proxy/management_endpoints/oracle_router_endpoints.py b/litellm/proxy/management_endpoints/oracle_router_endpoints.py index 10833105442..2361d06ea13 100644 --- a/litellm/proxy/management_endpoints/oracle_router_endpoints.py +++ b/litellm/proxy/management_endpoints/oracle_router_endpoints.py @@ -10,6 +10,7 @@ from typing import TYPE_CHECKING, Annotated, Final from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.router_strategy.oracle_router.oracle_router import VerificationBacklogFull from litellm.types.management_endpoints.oracle_router_endpoints import ( OracleRouterFeedbackRequest, OracleRouterFeedbackResponse, @@ -83,7 +84,12 @@ async def submit_oracle_router_feedback( status_code=404, detail={"error": f"Unknown or already completed program {data.program_id!r}."} ) holder, binding = found - holder.complete(data.program_id, score=data.score, cost=data.cost, payload=data.payload, owner=binding.api_key_hash) + try: + holder.complete( + data.program_id, score=data.score, cost=data.cost, payload=data.payload, owner=binding.api_key_hash + ) + except VerificationBacklogFull as error: + raise HTTPException(status_code=503, detail={"error": str(error)}) from error return OracleRouterFeedbackResponse( program_id=data.program_id, router_name=holder.router_name, diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index 140e9dd67fb..6b8c2de1f10 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -56,6 +56,13 @@ whose final response has no text, rather than reading a judge's skip as a pass. (admin) shows bindings, decision-maker state and recent feedback. The response header `x-litellm-oracle-router-model` names the bound model. +Verification runs in the background and is bounded: at most `max_pending_verifications` (default 64) +verifications are in flight per router, and each one is given `verification_timeout_seconds` (default +120) before it counts as failed. While the backlog is full a completion is refused and the program stays +bound, so it can be completed again once the backlog drains: the feedback endpoint answers 503 and a +`program_done` request records the refusal in `last_failure`; `verifications_rejected` in the state +counts them. + ## Security A program is private to the API key that started it: the same `program_id` under another key is another @@ -65,6 +72,10 @@ requests the Router stamped as routed by this router (`routing_decision`), so OR metadata of a request to some other model are ignored, and the router drops any caller-supplied copy of its internal metadata keys before stamping its own. +A `pre_routing` decision maker forwards the program's `turn_off_message_logging` opt-out to the strategy +router it wraps, so a classifier call made on the program's behalf never logs a prompt the caller marked +private. + `custom` decision makers and verifiers import Python named by the deployment, so they are honoured only for deployments in the proxy config file. A model written through the API (`/model/new`, `/model/update`, whatever the caller's role) with a `custom` type is rejected when the router is built and never imported. diff --git a/litellm/router_strategy/oracle_router/decision.py b/litellm/router_strategy/oracle_router/decision.py index 7a6ea12e72c..47a23cb467c 100644 --- a/litellm/router_strategy/oracle_router/decision.py +++ b/litellm/router_strategy/oracle_router/decision.py @@ -14,6 +14,7 @@ from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass from typing import Final, Protocol, runtime_checkable +from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy from litellm.router_strategy.adaptive_router.bandit import BanditCell, apply_delta, initial_cell, pick_best from litellm.types.router import ( AdaptiveRouterPreferences, @@ -31,6 +32,7 @@ class ProgramContext: program_id: str prompt: str request_type: RequestType + message_logging_off: bool = False @runtime_checkable @@ -164,11 +166,12 @@ class PreRoutingDecisionMaker: return self._models async def select(self, context: ProgramContext) -> str: - response: Final = await self._strategy.async_pre_routing_hook( - model=self._router_name, - request_kwargs={"metadata": {}}, - messages=[{"role": "user", "content": context.prompt}], - ) + with inherit_message_logging_privacy(context.message_logging_off): + response: Final = await self._strategy.async_pre_routing_hook( + model=self._router_name, + request_kwargs={"metadata": {"turn_off_message_logging": True} if context.message_logging_off else {}}, + messages=[{"role": "user", "content": context.prompt}], + ) proposed: Final = response.model if response is not None and response.model in self._models else None if proposed is None: self._fallbacks += 1 diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index 0be99e0f1c3..08abdf83ab3 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -21,6 +21,7 @@ from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict ) +from litellm.litellm_core_utils.internal_call_metadata import effective_turn_off_message_logging from litellm.router_strategy.adaptive_router.classifier import classify_prompt from litellm.router_strategy.oracle_router.config import ( CHOSEN_MODEL_METADATA_KEY, @@ -166,6 +167,10 @@ def _stamp_internal( internal[key] = value +class VerificationBacklogFull(RuntimeError): + """``max_pending_verifications`` verifications are in flight; the program stays bound for a later retry.""" + + class OracleRouter: """One instance per ``auto_router/oracle_router`` deployment.""" @@ -195,6 +200,7 @@ class OracleRouter: self.programs_evicted: int = 0 self.stateless_requests: int = 0 self.verifications_failed: int = 0 + self.verifications_rejected: int = 0 self.last_failure: str | None = None @property @@ -210,8 +216,13 @@ class OracleRouter: """ self.last_failure = f"{step}: {type(error).__name__}: {error}" - def context_for(self, program_id: str, prompt: str) -> ProgramContext: - return ProgramContext(program_id=program_id, prompt=prompt, request_type=classify_prompt(prompt)) + def context_for(self, program_id: str, prompt: str, request_kwargs: Mapping[str, object]) -> ProgramContext: + return ProgramContext( + program_id=program_id, + prompt=prompt, + request_type=classify_prompt(prompt), + message_logging_off=effective_turn_off_message_logging(request_kwargs) is True, + ) def binding(self, program_id: str, owner: str | None = None) -> ProgramBinding | None: """The program's row as seen by ``owner``, the key that started it (``user_api_key_hash``).""" @@ -233,7 +244,9 @@ class OracleRouter: self._bindings.pop(key, None) self.programs_evicted += len(expired) + len(oldest) - async def bind(self, program_id: str, prompt: str, api_key_hash: str | None) -> ProgramBinding: + async def bind( + self, program_id: str, prompt: str, api_key_hash: str | None, request_kwargs: Mapping[str, object] = _EMPTY + ) -> ProgramBinding: """Bind a new program, or return its existing row. A program is private to the key that started it: the same id under another key starts that key's @@ -252,16 +265,16 @@ class OracleRouter: event: Final = asyncio.Event() self._binding_in_flight[key] = event try: - return await self._bind(key, prompt) + return await self._bind(key, prompt, request_kwargs) finally: self._binding_in_flight.pop(key, None) event.set() - async def _bind(self, key: ProgramKey, prompt: str) -> ProgramBinding: + async def _bind(self, key: ProgramKey, prompt: str, request_kwargs: Mapping[str, object]) -> ProgramBinding: now: Final = self._clock() if len(self._bindings) >= min(PROGRAM_SWEEP_THRESHOLD, self.config.max_programs): self._evict_expired(now) - context: Final = self.context_for(key.program_id, prompt) + context: Final = self.context_for(key.program_id, prompt, request_kwargs) model: Final = await self.decision_maker.select(context) binding: Final = ProgramBinding( program_id=key.program_id, model=model, context=context, api_key_hash=key.owner, bound_at=now @@ -281,7 +294,7 @@ class OracleRouter: """Route the request to its program's model, binding the program on its first request.""" program_id: Final = resolve_program_id(request_kwargs, self.config.program_id_key) prompt: Final = first_user_text(messages) - chosen_model, request_type = await self._decide(program_id, prompt, _api_key_hash(request_kwargs)) + chosen_model, request_type = await self._decide(program_id, prompt, request_kwargs) _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) return PreRoutingHookResponse( @@ -296,12 +309,14 @@ class OracleRouter: ), ) - async def _decide(self, program_id: str | None, prompt: str, api_key_hash: str | None) -> tuple[str, RequestType]: + async def _decide( + self, program_id: str | None, prompt: str, request_kwargs: Mapping[str, object] + ) -> tuple[str, RequestType]: if program_id is None: self.stateless_requests += 1 - context: Final = self.context_for("", prompt) + context: Final = self.context_for("", prompt, request_kwargs) return await self.decision_maker.select(context), context.request_type - binding: Final = await self.bind(program_id, prompt, api_key_hash) + binding: Final = await self.bind(program_id, prompt, _api_key_hash(request_kwargs), request_kwargs) return binding.model, binding.context.request_type def observe_request( @@ -337,6 +352,9 @@ class OracleRouter: A program none of whose requests has been observed yet (feedback that raced the last response's logging, or a program whose only request failed) keeps its slot and is completed, with these arguments, by its next observed response: there is no answer to verify yet, so nothing is learned. + + Raises ``VerificationBacklogFull`` when ``max_pending_verifications`` verifications are already in + flight: the program stays bound, so the caller can complete it again once the backlog drains. """ key: Final = ProgramKey(owner, program_id) binding: Final = self._bindings.get(key) @@ -345,6 +363,12 @@ class OracleRouter: if binding.requests == 0: self._bindings[key] = replace(binding, deferred=DeferredCompletion(score=score, cost=cost, payload=payload)) return None + if len(self._tasks) >= self.config.max_pending_verifications: + self.verifications_rejected += 1 + raise VerificationBacklogFull( + f"{len(self._tasks)} verifications are in flight (max_pending_verifications=" + f"{self.config.max_pending_verifications}); program {program_id!r} stays bound, complete it again later" + ) del self._bindings[key] outcome: Final = ProgramOutcome( program_id=program_id, @@ -364,7 +388,9 @@ class OracleRouter: async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float: try: - score: Final = clamp_score(await self.verifier.verify(outcome)) + score: Final = clamp_score( + await asyncio.wait_for(self.verifier.verify(outcome), timeout=self.config.verification_timeout_seconds) + ) except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop self.verifications_failed += 1 self.note_failure("verification", error) @@ -416,6 +442,7 @@ class OracleRouter: "stateless_requests": self.stateless_requests, "pending_verifications": self.pending_verifications, "verifications_failed": self.verifications_failed, + "verifications_rejected": self.verifications_rejected, "last_failure": self.last_failure, "recent_accuracy": self._accuracy_by_model(), "recent_feedback": [ diff --git a/litellm/types/router.py b/litellm/types/router.py index 007489e5281..2ca2b411a21 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -1597,6 +1597,8 @@ class OracleRouterConfig(BaseModel, frozen=True): program_id_key: str = "program_id" program_ttl_seconds: int = Field(default=24 * 3600, ge=1) max_programs: int = Field(default=10_000, ge=1) + max_pending_verifications: int = Field(default=64, ge=1) + verification_timeout_seconds: float = Field(default=120.0, gt=0) @model_validator(mode="after") def _fixed_model_is_available(self) -> "OracleRouterConfig": diff --git a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py index e4bcfa92d2a..da9236b2c6c 100644 --- a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py @@ -1,5 +1,6 @@ """POST /oracle_router/feedback and GET /oracle_router/state.""" +import asyncio from unittest.mock import patch import pytest @@ -91,6 +92,28 @@ async def test_feedback_completes_the_program_and_the_router_learns(oracle): assert oracle.maker.updates == [("smart", 1.0)] +@pytest.mark.asyncio +async def test_feedback_is_rejected_with_503_while_the_verification_backlog_is_full(): + class _Stuck: + async def verify(self, outcome): + await asyncio.sleep(5.0) + return 1.0 + + config = OracleRouterConfig(available_models=["smart", "fast"], max_pending_verifications=1) + oracle = OracleRouter("oracle", config, _Maker(), _Stuck()) + for program_id in ("task-1", "task-2"): + await _bind(oracle, program_id) + oracle.observe_request(program_id, cost=0.2, response_text="done", owner="owner-hash") + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER) + with pytest.raises(HTTPException) as rejected: + await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-2", score=1.0), OWNER) + assert rejected.value.status_code == 503 and "max_pending_verifications=1" in rejected.value.detail["error"] + assert oracle.binding("task-2", "owner-hash") is not None + for task in tuple(oracle._tasks): + task.cancel() + + @pytest.mark.asyncio async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it(oracle): await _bind(oracle, "task-1") diff --git a/tests/unit/router_strategy/oracle_router/test_decision.py b/tests/unit/router_strategy/oracle_router/test_decision.py index 5e17d3fc9c8..3e89e13f01d 100644 --- a/tests/unit/router_strategy/oracle_router/test_decision.py +++ b/tests/unit/router_strategy/oracle_router/test_decision.py @@ -2,6 +2,10 @@ import pytest +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, +) +from litellm.litellm_core_utils.internal_call_metadata import effective_turn_off_message_logging from litellm.router_strategy.oracle_router.decision import ( DecisionMaker, FixedDecisionMaker, @@ -90,6 +94,10 @@ class _FakeStrategy: async def async_pre_routing_hook(self, model, request_kwargs, messages=None, input=None, specific_deployment=False): self.calls.append({"model": model, "messages": messages}) + self.privacy = ( + effective_turn_off_message_logging(request_kwargs), + initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging"), + ) return None if self.answer is None else PreRoutingHookResponse(model=self.answer, messages=messages) @@ -102,6 +110,19 @@ async def test_pre_routing_uses_the_wrapped_strategy_proposal_once_per_program() assert maker.snapshot()["proposals"] == {"fast": 1} +@pytest.mark.asyncio +async def test_pre_routing_forwards_the_program_s_message_logging_opt_out_to_the_wrapped_strategy(): + strategy = _FakeStrategy("fast") + maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy) + private = ProgramContext( + program_id="p", prompt="secret", request_type=RequestType.GENERAL, message_logging_off=True + ) + assert await maker.select(private) == "fast" + assert strategy.privacy == (True, True) + assert await maker.select(_ctx("public")) == "fast" + assert strategy.privacy == (None, None) + + @pytest.mark.asyncio async def test_pre_routing_falls_back_to_the_first_model_when_the_proposal_is_unusable(): assert await PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy(None)).select(_ctx()) == "smart" diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index f5651cc8b48..4bb717fa033 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -1,6 +1,7 @@ """OracleRouter: sticky program bindings, program id resolution, delayed feedback, eviction.""" import asyncio +import math import pytest @@ -10,7 +11,12 @@ from litellm.router_strategy.oracle_router.config import ( PROGRAM_ID_METADATA_KEY, ) from litellm.router_strategy.oracle_router.decision import ProgramContext -from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, first_user_text, resolve_program_id +from litellm.router_strategy.oracle_router.oracle_router import ( + OracleRouter, + VerificationBacklogFull, + first_user_text, + resolve_program_id, +) from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, ReportedVerifier from litellm.types.router import OracleRouterConfig @@ -181,6 +187,54 @@ async def test_complete_releases_the_slot_now_and_learns_in_verification_order() assert snapshot["recent_feedback"][-1]["cost"] == pytest.approx(0.6) +@pytest.mark.asyncio +async def test_a_full_verification_backlog_rejects_completion_and_keeps_the_program_bound(): + router, maker = _router(verifier=_SlowVerifier({"t1": 0.05}), max_pending_verifications=1) + for program_id in ("t1", "t2"): + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages() + ) + router.observe_request(program_id, cost=0.1, response_text="answer") + first = router.complete("t1", score=1.0) + with pytest.raises(VerificationBacklogFull, match="max_pending_verifications=1"): + router.complete("t2", score=0.0) + assert router.binding("t2") is not None and router.pending_verifications == 1 + assert router.verifications_rejected == 1 and router.programs_completed == 1 + assert await first == 1.0 + assert await router.complete("t2", score=0.0) == 0.0 + assert router.binding("t2") is None and [u[0] for u in maker.updates] == ["t1", "t2"] + assert (await router.get_state_snapshot())["verifications_rejected"] == 1 + + +@pytest.mark.asyncio +async def test_a_verification_that_outlives_its_timeout_fails_and_teaches_nothing(): + router, maker = _router(verifier=_SlowVerifier({"t": 5.0}), verification_timeout_seconds=0.01) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + router.observe_request("t", cost=0.1, response_text="answer") + task = router.complete("t", score=1.0) + assert task is not None and math.isnan(await task) + assert router.verifications_failed == 1 and maker.updates == [] + assert router.last_failure is not None and "verification: TimeoutError" in router.last_failure + assert router.pending_verifications == 0 + + +@pytest.mark.asyncio +async def test_a_program_s_message_logging_opt_out_is_kept_on_its_context(): + router, _ = _router() + await router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "private", "turn_off_message_logging": True}}, + messages=_messages(), + ) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "public"}}, messages=_messages() + ) + assert router.binding("private").context.message_logging_off is True + assert router.binding("public").context.message_logging_off is False + + @pytest.mark.asyncio async def test_complete_cost_override_and_payload_reach_the_verifier(): seen: list[ProgramOutcome] = [] @@ -216,7 +270,9 @@ async def test_failed_verification_is_counted_and_does_not_update(): result = await router.complete("t") assert result != result and router.verifications_failed == 1 and maker.updates == [] snapshot = await router.get_state_snapshot() - assert snapshot["last_failure"] == "verification: RuntimeError: sandbox down" # kept for the state endpoint, not logged + assert ( + snapshot["last_failure"] == "verification: RuntimeError: sandbox down" + ) # kept for the state endpoint, not logged @pytest.mark.asyncio From 67e5637f63e5dc96c8354b16b10601af4d68948c Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Thu, 5 Nov 2026 00:40:10 +0000 Subject: [PATCH 10/10] fix(oracle_router): bill a program's classifier call to its caller and verify private programs privately A pre_routing decision maker called the wrapped strategy with only a privacy flag, so an LLM-backed strategy made its classifier call without the caller's identity and the proxy's cost callback skipped its spend. The program context now carries what LiteLLM forwards to any detached sub-call, the caller's identity minus the budget reservation through sanitized_forwardable_call_metadata, and the wrapped strategy receives it in a fresh metadata dict; it is used for that call only and never kept on the binding. Verification ignored the program's message-logging opt-out and the judge's child completion carries only its origin, so private prompts and transcripts reached logging callbacks. The opt-out is now derived with the logger's own redaction rule at bind time and on every observed turn, success or failure, kept on the binding and the outcome, and the verifier runs under inherit_message_logging_privacy. A guardrail verifier refuses a final turn that LiteLLM already redacted instead of judging the placeholder, the feedback payload reaches a guardrail without proxy-owned identity keys, and a Responses API program is bound on its converted input text instead of an empty prompt. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- .../router_strategy/oracle_router/README.md | 19 +++- .../router_strategy/oracle_router/decision.py | 17 ++- .../router_strategy/oracle_router/hooks.py | 4 +- .../oracle_router/oracle_router.py | 102 ++++++++++++++++-- .../router_strategy/oracle_router/verifier.py | 27 ++++- .../oracle_router/test_decision.py | 18 ++++ .../oracle_router/test_hooks.py | 28 +++++ .../oracle_router/test_oracle_router.py | 92 ++++++++++++++++ .../oracle_router/test_verifier.py | 42 ++++++++ 9 files changed, 332 insertions(+), 17 deletions(-) diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index 6b8c2de1f10..44d4200a164 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -72,9 +72,22 @@ requests the Router stamped as routed by this router (`routing_decision`), so OR metadata of a request to some other model are ignored, and the router drops any caller-supplied copy of its internal metadata keys before stamping its own. -A `pre_routing` decision maker forwards the program's `turn_off_message_logging` opt-out to the strategy -router it wraps, so a classifier call made on the program's behalf never logs a prompt the caller marked -private. +A `pre_routing` decision maker hands the strategy router it wraps what LiteLLM forwards to any detached +sub-call: the caller's identity (`user_api_key`, its hash and alias, team, user, organization and end user, +and the auth object the Router filters deployments by, minus the request's budget reservation), nothing else +of the request, and the program's `turn_off_message_logging` opt-out. A classifier call made on the program's +behalf is therefore billed to the caller's key and team and never logs a prompt the caller marked private; +the identity is used for that call only and is not kept on the program. Verification runs under the +program's opt-out as well: a program is private once any of its requests opted out, whether or not that +request produced an answer, so a `guardrail` verifier's judge call logs neither the prompt nor the +transcript. LiteLLM redacts a private request's messages and answer before the router's hook sees them, so +a `guardrail` verifier cannot judge a program whose final turn was redacted: that is a failed verification +and teaches the decision maker nothing. Use the `reported` verifier for routers whose programs opt out of +logging. A `guardrail` verifier's judge call carries no caller identity, as the proxy's own +`llm_as_a_judge` flow does not, so its spend is not attributed to the program's key or team. The feedback +`payload` reaches the guardrail with the proxy-owned keys removed (`user_api_key_*`, the sub-call origin, +headers and the routing decision), so a caller cannot pose as the proxy to a guardrail that forwards +identity to its vendor. `custom` decision makers and verifiers import Python named by the deployment, so they are honoured only for deployments in the proxy config file. A model written through the API (`/model/new`, `/model/update`, diff --git a/litellm/router_strategy/oracle_router/decision.py b/litellm/router_strategy/oracle_router/decision.py index 47a23cb467c..d31f1438a5a 100644 --- a/litellm/router_strategy/oracle_router/decision.py +++ b/litellm/router_strategy/oracle_router/decision.py @@ -11,7 +11,8 @@ import importlib import itertools import random from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field +from types import MappingProxyType from typing import Final, Protocol, runtime_checkable from litellm.litellm_core_utils.initialize_dynamic_callback_params import inherit_message_logging_privacy @@ -24,6 +25,8 @@ from litellm.types.router import ( RequestType, ) +_NO_CALLER: Final[Mapping[str, object]] = MappingProxyType({}) + @dataclass(frozen=True, slots=True) class ProgramContext: @@ -33,6 +36,8 @@ class ProgramContext: prompt: str request_type: RequestType message_logging_off: bool = False + caller: Mapping[str, object] = field(default_factory=lambda: _NO_CALLER) + """The caller's identity metadata, so a sub-call made on the program's behalf is billed to the caller.""" @runtime_checkable @@ -143,6 +148,14 @@ class FixedDecisionMaker: return {"type": "fixed", "model": self._model, "updates": self._updates} +def strategy_request_kwargs(context: ProgramContext) -> dict[str, object]: # mutable-ok: pre-routing contract + """The request a wrapped strategy router sees for the program: the caller's identity, so a classifier call + the strategy makes is billed to the caller's key and team, and the program's message-logging opt-out where + LiteLLM reads it. A fresh dict per call: the strategies write their decision into the bucket they are handed.""" + privacy: Final[Mapping[str, object]] = {"turn_off_message_logging": True} if context.message_logging_off else {} + return {"metadata": {**context.caller, **privacy}} + + class PreRoutingDecisionMaker: """Let one of LiteLLM's own strategy routers propose the model, once per program. @@ -169,7 +182,7 @@ class PreRoutingDecisionMaker: with inherit_message_logging_privacy(context.message_logging_off): response: Final = await self._strategy.async_pre_routing_hook( model=self._router_name, - request_kwargs={"metadata": {"turn_off_message_logging": True} if context.message_logging_off else {}}, + request_kwargs=strategy_request_kwargs(context), messages=[{"role": "user", "content": context.prompt}], ) proposed: Final = response.model if response is not None and response.model in self._models else None diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py index 6165f6293e3..3d2f2ffc3c3 100644 --- a/litellm/router_strategy/oracle_router/hooks.py +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -19,7 +19,7 @@ from litellm.router_strategy.oracle_router.config import ( PROGRAM_SCORE_KEY, RESPONSE_HEADER, ) -from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, as_str_mapping +from litellm.router_strategy.oracle_router.oracle_router import OracleRouter, as_str_mapping, logged_request_is_private from litellm.router_strategy.oracle_router.verifier import response_text if TYPE_CHECKING: @@ -102,6 +102,8 @@ class OracleRouterPostCallHook(CustomLogger): if not isinstance(program_id, str): return owner: Final = _owner(metadata) + if logged_request_is_private(kwargs): + self.oracle_router.mark_message_logging_off(program_id, owner) if succeeded: cost: Final = kwargs.get("response_cost") self.oracle_router.observe_request( diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index 08abdf83ab3..2df10409035 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -21,7 +21,15 @@ from pydantic import TypeAdapter, ValidationError from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict ) -from litellm.litellm_core_utils.internal_call_metadata import effective_turn_off_message_logging +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + inherit_message_logging_privacy, + initialize_standard_callback_dynamic_params, +) +from litellm.litellm_core_utils.internal_call_metadata import sanitized_forwardable_call_metadata +from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages +from litellm.litellm_core_utils.redact_messages import ( + should_redact_message_logging, # pyright: ignore[reportUnknownVariableType] # upstream helper takes a bare dict +) from litellm.router_strategy.adaptive_router.classifier import classify_prompt from litellm.router_strategy.oracle_router.config import ( CHOSEN_MODEL_METADATA_KEY, @@ -35,7 +43,11 @@ from litellm.router_strategy.oracle_router.decision import DecisionMaker, Progra from litellm.router_strategy.oracle_router.verifier import ProgramOutcome, Verifier, clamp_score from litellm.types.llms.openai import AllMessageValues from litellm.types.router import OracleRouterConfig, PreRoutingHookResponse, RequestType -from litellm.types.utils import RoutingDecisionCause, StandardLoggingRoutingDecision +from litellm.types.utils import ( + AUTOROUTER_CLASSIFIER_CALL_ORIGIN, + RoutingDecisionCause, + StandardLoggingRoutingDecision, +) _LEARNING_DECISION_MAKERS: Final[frozenset[str]] = frozenset({"thompson"}) _MESSAGES: Final = TypeAdapter(Sequence[AllMessageValues]) @@ -84,6 +96,8 @@ class ProgramBinding: last_response_text: str = "" last_messages: Sequence[AllMessageValues] = () deferred: DeferredCompletion | None = None + message_logging_off: bool = False + """True once any request of the program opted out of message logging; verification then runs privately.""" @dataclass(frozen=True, slots=True) @@ -96,6 +110,21 @@ class FeedbackRecord: verified_at: float +def program_messages( + messages: list[dict[str, object]] | None, # mutable-ok: pre-routing contract + input: str | list[object] | None, # mutable-ok: pre-routing contract + request_kwargs: Mapping[str, object], +) -> Sequence[Mapping[str, object]] | None: + """The request's messages in chat shape whichever surface sent them: a Responses API ``input`` is converted + the way guardrails convert it, so a program started through ``/v1/responses`` is bound on its real prompt.""" + if messages or input is None: + return messages + return resolve_structured_messages( + None, + {**request_kwargs, "input": input, "litellm_metadata": as_str_mapping(request_kwargs.get("litellm_metadata"))}, + ) + + def first_user_text(messages: Sequence[Mapping[str, object]] | None) -> str: """The program's initial request: the first user message, text parts only.""" for message in messages or (): @@ -143,6 +172,45 @@ def _transcript(messages: object, fallback: Sequence[AllMessageValues]) -> Seque return fallback +def request_is_private(request_kwargs: Mapping[str, object]) -> bool: + """Whether LiteLLM will redact this request's messages when it logs them, by the logger's own rule: the + request's ``turn_off_message_logging``, its redaction headers, or the global setting.""" + kwargs: Final = dict(request_kwargs) + return _redacts( + { + "litellm_params": kwargs, + "standard_callback_dynamic_params": initialize_standard_callback_dynamic_params(kwargs), + } + ) + + +def logged_request_is_private(kwargs: Mapping[str, object]) -> bool: + """Whether LiteLLM redacted a logged request's messages, from the kwargs it hands a logging callback.""" + return _redacts( + { + **kwargs, + "standard_callback_dynamic_params": kwargs.get("standard_callback_dynamic_params") + or initialize_standard_callback_dynamic_params(dict(kwargs)), + } + ) + + +def _redacts(model_call_details: dict[str, object]) -> bool: # mutable-ok: the logger's own parameter shape + try: + return should_redact_message_logging(model_call_details) + except AttributeError: # malformed metadata makes the logger raise and log nothing, so the program stays private + return True + + +def forwardable_caller(request_kwargs: Mapping[str, object]) -> Mapping[str, object]: + """What a classifier call made on the program's behalf inherits from the request that bound it: the caller's + identity from whichever metadata bucket the surface used, with its budget reservation removed, stamped as the + sub-call it becomes. The routing decision, the logging payload and every other per-request state stay behind.""" + return MappingProxyType( + sanitized_forwardable_call_metadata(_metadata(request_kwargs), AUTOROUTER_CLASSIFIER_CALL_ORIGIN) + ) + + def _api_key_hash(request_kwargs: Mapping[str, object]) -> str | None: """The key the proxy authenticated, stamped server-side as ``user_api_key_hash``; None outside the proxy.""" value: Final = _metadata(request_kwargs).get("user_api_key_hash") @@ -221,7 +289,8 @@ class OracleRouter: program_id=program_id, prompt=prompt, request_type=classify_prompt(prompt), - message_logging_off=effective_turn_off_message_logging(request_kwargs) is True, + message_logging_off=request_is_private(request_kwargs), + caller=forwardable_caller(request_kwargs), ) def binding(self, program_id: str, owner: str | None = None) -> ProgramBinding | None: @@ -277,7 +346,12 @@ class OracleRouter: context: Final = self.context_for(key.program_id, prompt, request_kwargs) model: Final = await self.decision_maker.select(context) binding: Final = ProgramBinding( - program_id=key.program_id, model=model, context=context, api_key_hash=key.owner, bound_at=now + program_id=key.program_id, + model=model, + context=replace(context, caller=_EMPTY), + api_key_hash=key.owner, + bound_at=now, + message_logging_off=context.message_logging_off, ) self._bindings[key] = binding self.programs_bound += 1 @@ -293,7 +367,7 @@ class OracleRouter: ) -> PreRoutingHookResponse | None: """Route the request to its program's model, binding the program on its first request.""" program_id: Final = resolve_program_id(request_kwargs, self.config.program_id_key) - prompt: Final = first_user_text(messages) + prompt: Final = first_user_text(program_messages(messages, input, request_kwargs)) chosen_model, request_type = await self._decide(program_id, prompt, request_kwargs) _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) @@ -319,6 +393,14 @@ class OracleRouter: binding: Final = await self.bind(program_id, prompt, _api_key_hash(request_kwargs), request_kwargs) return binding.model, binding.context.request_type + def mark_message_logging_off(self, program_id: str, owner: str | None = None) -> None: + """A request of the program opted out of message logging, whether or not it produced an answer: the + whole program is private from here on, and its verification runs under that opt-out.""" + key: Final = ProgramKey(owner, program_id) + binding: Final = self._bindings.get(key) + if binding is not None and not binding.message_logging_off: + self._bindings[key] = replace(binding, message_logging_off=True) + def observe_request( self, program_id: str, cost: float, response_text: str, messages: object = None, owner: str | None = None ) -> None: @@ -379,6 +461,7 @@ class OracleRouter: messages=binding.last_messages, score=score, payload=payload, + message_logging_off=binding.message_logging_off, ) self.programs_completed += 1 task: Final = asyncio.create_task(self._verify_and_learn(binding, outcome)) @@ -388,9 +471,12 @@ class OracleRouter: async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float: try: - score: Final = clamp_score( - await asyncio.wait_for(self.verifier.verify(outcome), timeout=self.config.verification_timeout_seconds) - ) + with inherit_message_logging_privacy(outcome.message_logging_off): + score: Final = clamp_score( + await asyncio.wait_for( + self.verifier.verify(outcome), timeout=self.config.verification_timeout_seconds + ) + ) except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop self.verifications_failed += 1 self.note_failure("verification", error) diff --git a/litellm/router_strategy/oracle_router/verifier.py b/litellm/router_strategy/oracle_router/verifier.py index 5b202d90701..7a52ae6b2c0 100644 --- a/litellm/router_strategy/oracle_router/verifier.py +++ b/litellm/router_strategy/oracle_router/verifier.py @@ -14,6 +14,7 @@ from typing import Final, Protocol, runtime_checkable from pydantic import TypeAdapter, ValidationError import litellm +from litellm.constants import INTERNAL_CALL_ORIGIN_METADATA_KEY, REDACTED_BY_LITELLM from litellm.exceptions import GuardrailRaisedException from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.types.llms.openai import AllMessageValues @@ -26,6 +27,9 @@ _EMPTY: Final[Mapping[str, object]] = MappingProxyType({}) _STR_MAPPING: Final = TypeAdapter(Mapping[str, object]) _STATUS_RECORDS: Final = TypeAdapter(Sequence[Mapping[str, object]]) _STATUS_RECORD_KEY: Final[str] = "standard_logging_guardrail_information" +_PROXY_OWNED_METADATA_KEYS: Final[frozenset[str]] = frozenset( + {_STATUS_RECORD_KEY, INTERNAL_CALL_ORIGIN_METADATA_KEY, "headers", "routing_decision"} +) _BLOCKED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_intervened", "guardrail_flagged"}) _UNVERIFIED_STATUSES: Final[frozenset[str]] = frozenset({"guardrail_failed_to_respond", "not_run"}) @@ -42,6 +46,8 @@ class ProgramOutcome: messages: Sequence[AllMessageValues] = () score: float | None = None payload: Mapping[str, object] = field(default_factory=lambda: _EMPTY) + message_logging_off: bool = False + """Whether any request of the program opted out of message logging; the router verifies under that opt-out.""" @runtime_checkable @@ -82,19 +88,31 @@ def find_guardrail(name: str) -> CustomGuardrail | None: return next((guardrail for guardrail in guardrails if guardrail.guardrail_name == name), None) +def _redacted(outcome: ProgramOutcome) -> bool: + """Whether LiteLLM replaced the program's final answer or transcript with its redaction placeholder.""" + return outcome.response_text == REDACTED_BY_LITELLM or any( + _STR_MAPPING.validate_python(message).get("content") == REDACTED_BY_LITELLM for message in outcome.messages + ) + + def _verifier_metadata(outcome: ProgramOutcome) -> dict[str, object]: # mutable-ok: the guardrail writes into it """The feedback payload as the guardrail's request metadata. The verdict is read from the status record the guardrail writes here, so the caller's payload must not - pre-seed that key (a non-record entry would make the whole list unreadable and hide a rejection), and - it cannot rename the program. + pre-seed that key (a non-record entry would make the whole list unreadable and hide a rejection), it + cannot rename the program, and it cannot pose as the proxy: the ``user_api_key_*`` identity a guardrail + may forward to its vendor, the sub-call origin and the request headers are the proxy's to stamp. """ return { - **{key: value for key, value in outcome.payload.items() if key != _STATUS_RECORD_KEY}, + **{key: value for key, value in outcome.payload.items() if not _proxy_owned(key)}, "program_id": outcome.program_id, } +def _proxy_owned(key: str) -> bool: + return key.startswith("user_api_key") or key in _PROXY_OWNED_METADATA_KEYS + + def _status_records(bucket: object) -> Sequence[Mapping[str, object]]: """The ``standard_logging_guardrail_information`` records of one metadata bucket, or none.""" try: @@ -173,6 +191,9 @@ class GuardrailVerifier: if not outcome.response_text.strip(): # A judge skips an empty answer without recording a verdict; that must not read as a pass. raise RuntimeError(f"guardrail {self._name!r}: the program has no response text to verify") + if _redacted(outcome): + # The request opted out of message logging, so the router only ever saw LiteLLM's placeholder. + raise RuntimeError(f"guardrail {self._name!r}: the program's final turn was redacted before verification") transcript: Final = list(outcome.messages) or [{"role": "user", "content": outcome.prompt}] inputs: Final = GenericGuardrailAPIInputs( texts=[outcome.response_text], model=outcome.model, structured_messages=transcript diff --git a/tests/unit/router_strategy/oracle_router/test_decision.py b/tests/unit/router_strategy/oracle_router/test_decision.py index 3e89e13f01d..77f4bdfc537 100644 --- a/tests/unit/router_strategy/oracle_router/test_decision.py +++ b/tests/unit/router_strategy/oracle_router/test_decision.py @@ -94,6 +94,7 @@ class _FakeStrategy: async def async_pre_routing_hook(self, model, request_kwargs, messages=None, input=None, specific_deployment=False): self.calls.append({"model": model, "messages": messages}) + self.request_kwargs = request_kwargs self.privacy = ( effective_turn_off_message_logging(request_kwargs), initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging"), @@ -123,6 +124,23 @@ async def test_pre_routing_forwards_the_program_s_message_logging_opt_out_to_the assert strategy.privacy == (None, None) +@pytest.mark.asyncio +async def test_pre_routing_forwards_the_caller_s_identity_so_a_classifier_call_is_billed_to_the_caller(): + strategy = _FakeStrategy("fast") + maker = PreRoutingDecisionMaker(MODELS, router_name="tiered", strategy=strategy) + caller = {"user_api_key_hash": "hash-1", "user_api_key_team_id": "team-1", "user_api_key_user_id": "user-1"} + private = ProgramContext( + program_id="p", prompt="task", request_type=RequestType.GENERAL, message_logging_off=True, caller=caller + ) + assert await maker.select(private) == "fast" + assert strategy.request_kwargs == {"metadata": {**caller, "turn_off_message_logging": True}} + first_call = strategy.request_kwargs + assert await maker.select(private) == "fast" + assert strategy.request_kwargs == first_call and strategy.request_kwargs is not first_call + assert await maker.select(_ctx()) == "fast" + assert strategy.request_kwargs == {"metadata": {}} + + @pytest.mark.asyncio async def test_pre_routing_falls_back_to_the_first_model_when_the_proposal_is_unusable(): assert await PreRoutingDecisionMaker(MODELS, "r", _FakeStrategy(None)).select(_ctx()) == "smart" diff --git a/tests/unit/router_strategy/oracle_router/test_hooks.py b/tests/unit/router_strategy/oracle_router/test_hooks.py index 82e9710d755..55299fc72ec 100644 --- a/tests/unit/router_strategy/oracle_router/test_hooks.py +++ b/tests/unit/router_strategy/oracle_router/test_hooks.py @@ -77,6 +77,34 @@ async def test_success_events_accumulate_spend_and_the_latest_answer(bound): assert list(binding.last_messages) == [{"role": "user", "content": "task"}] +@pytest.mark.asyncio +async def test_a_turn_that_opted_out_of_message_logging_marks_the_program_private_for_good(bound): + router, _, hook = bound + await hook.async_log_success_event(_kwargs(cost=0.01), _response("first"), None, None) + assert router.binding("t1").message_logging_off is False + await hook.async_log_success_event( + _kwargs(cost=0.01, turn_off_message_logging=True), _response("second"), None, None + ) + assert router.binding("t1").message_logging_off is True + await hook.async_log_success_event(_kwargs(cost=0.01), _response("third"), None, None) + assert router.binding("t1").message_logging_off is True + + +@pytest.mark.asyncio +async def test_a_private_turn_that_failed_still_marks_the_program_private(bound): + router, _, hook = bound + await hook.async_log_failure_event(_kwargs(cost=0.5, turn_off_message_logging=True), None, None, None) + assert router.binding("t1").message_logging_off is True and router.binding("t1").requests == 0 + + +@pytest.mark.asyncio +async def test_a_turn_litellm_resolved_as_private_marks_the_program_private(bound): + router, _, hook = bound + resolved = {**_kwargs(cost=0.01), "standard_callback_dynamic_params": {"turn_off_message_logging": True}} + await hook.async_log_success_event(resolved, _response("first"), None, None) + assert router.binding("t1").message_logging_off is True + + @pytest.mark.asyncio async def test_program_done_completes_the_program_with_the_reported_score(bound): router, maker, hook = bound diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index 4bb717fa033..0b51c6e3f9e 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -5,6 +5,10 @@ import math import pytest +from litellm.litellm_core_utils.core_helpers import budget_reservation_from_metadata +from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + initialize_standard_callback_dynamic_params, +) from litellm.router_strategy.oracle_router.config import ( CHOSEN_MODEL_METADATA_KEY, PROGRAM_ID_HEADER, @@ -29,6 +33,7 @@ class _RecordingMaker: def __init__(self) -> None: self.selections = 0 self.updates: list[tuple[str, str, float]] = [] + self.contexts: list[ProgramContext] = [] @property def models(self): @@ -36,6 +41,7 @@ class _RecordingMaker: async def select(self, context: ProgramContext) -> str: self.selections += 1 + self.contexts.append(context) return MODELS[(self.selections - 1) % 2] def update(self, context: ProgramContext, model: str, score: float) -> None: @@ -231,10 +237,96 @@ async def test_a_program_s_message_logging_opt_out_is_kept_on_its_context(): await router.async_pre_routing_hook( model="oracle", request_kwargs={"metadata": {"program_id": "public"}}, messages=_messages() ) + await router.async_pre_routing_hook( + model="oracle", + request_kwargs={ + "metadata": {"program_id": "by-header", "headers": {"litellm-enable-message-redaction": "true"}} + }, + messages=_messages(), + ) assert router.binding("private").context.message_logging_off is True + assert router.binding("by-header").context.message_logging_off is True assert router.binding("public").context.message_logging_off is False +@pytest.mark.asyncio +async def test_the_decision_maker_sees_the_caller_s_identity_and_none_of_the_request_s_other_state(): + router, maker = _router() + kwargs = { + "metadata": { + "program_id": "t", + "user_api_key_hash": "hash-1", + "user_api_key_team_id": "team-1", + "user_api_key_budget_reservation": {"amount": 1.0}, + "user_api_key_auth": {"api_key": "hash-1", "budget_reservation": {"amount": 1.0}}, + "routing_decision": {"router_type": "oracle"}, + "custom": "value", + }, + "litellm_metadata": {"user_api_key_user_id": "user-1", "_autorouter_baseline_route": "stamp"}, + } + await router.async_pre_routing_hook(model="oracle", request_kwargs=kwargs, messages=_messages()) + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"litellm_metadata": {"user_api_key_hash": "hash-2"}}, messages=_messages() + ) + bound, stateless = maker.contexts + assert bound.caller == { + "user_api_key_hash": "hash-1", + "user_api_key_team_id": "team-1", + "user_api_key_user_id": "user-1", + "user_api_key_auth": {"api_key": "hash-1"}, + "internal_call_origin": "autorouter_classifier", + } + assert budget_reservation_from_metadata(dict(bound.caller)) is None + assert stateless.caller == {"user_api_key_hash": "hash-2", "internal_call_origin": "autorouter_classifier"} + assert router.context_for("", "task", {"metadata": {"routing_decision": {}}}).caller == { + "internal_call_origin": "autorouter_classifier" + } + assert router.binding("t", "hash-1").context.caller == {} + + +@pytest.mark.asyncio +async def test_an_opt_out_on_a_later_turn_makes_the_whole_program_s_verification_private(): + seen: list[tuple[bool, bool | None]] = [] + + class _Peek: + async def verify(self, outcome: ProgramOutcome) -> float: + inherited = initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging") + seen.append((outcome.message_logging_off, inherited)) + return 1.0 + + router, _ = _router(verifier=_Peek()) + for program_id in ("public", "late-private"): + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": program_id}}, messages=_messages() + ) + router.observe_request("public", cost=0.1, response_text="a") + router.observe_request("late-private", cost=0.1, response_text="a") + router.mark_message_logging_off("late-private") + router.observe_request("late-private", cost=0.1, response_text="c") + assert router.binding("public").message_logging_off is False + assert router.binding("late-private").message_logging_off is True + assert await router.complete("public", score=1.0) == 1.0 + assert await router.complete("late-private", score=1.0) == 1.0 + assert seen == [(False, None), (True, True)] + assert initialize_standard_callback_dynamic_params({}).get("turn_off_message_logging") is None + + +@pytest.mark.asyncio +async def test_a_responses_api_request_binds_its_program_on_the_input_text(): + router, _ = _router() + response = await router.async_pre_routing_hook( + model="oracle", + request_kwargs={"metadata": {"program_id": "r"}}, + messages=None, + input=[{"role": "user", "content": [{"type": "input_text", "text": "write a limerick"}]}], + ) + assert response.messages is None and router.binding("r").context.prompt == "write a limerick" + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "s"}}, messages=None, input="plain text" + ) + assert router.binding("s").context.prompt == "plain text" + + @pytest.mark.asyncio async def test_complete_cost_override_and_payload_reach_the_verifier(): seen: list[ProgramOutcome] = [] diff --git a/tests/unit/router_strategy/oracle_router/test_verifier.py b/tests/unit/router_strategy/oracle_router/test_verifier.py index 66164f6b914..91018f2d4a0 100644 --- a/tests/unit/router_strategy/oracle_router/test_verifier.py +++ b/tests/unit/router_strategy/oracle_router/test_verifier.py @@ -3,6 +3,7 @@ import pytest from fastapi import HTTPException +from litellm.constants import REDACTED_BY_LITELLM from litellm.exceptions import GuardrailRaisedException from litellm.proxy.guardrails.guardrail_hooks.llm_as_a_judge import LLMAsAJudgeGuardrail from litellm.router_strategy.oracle_router.verifier import ( @@ -61,6 +62,47 @@ class _RecordingGuardrail: return inputs +@pytest.mark.asyncio +async def test_feedback_payload_cannot_pose_as_the_proxy_to_the_guardrail(): + judge = _RecordingGuardrail("success") + outcome = ProgramOutcome( + program_id="p", + model="smart", + prompt="task", + response_text="answer", + cost=0.1, + payload={ + "container": "c1", + "user_api_key_team_id": "someone-else", + "user_api_key": "sk-forged", + "internal_call_origin": "shadow_eval_judge", + "headers": {"litellm-disable-message-redaction": "true"}, + "routing_decision": {"router_type": "oracle"}, + }, + ) + assert await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(outcome) == 1.0 + assert judge.seen == {"container": "c1", "program_id": "p"} + + +@pytest.mark.asyncio +async def test_guardrail_verifier_refuses_a_final_answer_litellm_redacted_instead_of_judging_the_placeholder(): + judge = _Guardrail(blocks=False) + outcome = ProgramOutcome(program_id="p", model="smart", prompt="task", response_text=REDACTED_BY_LITELLM, cost=0.1) + with pytest.raises(RuntimeError, match="redacted before verification"): + await GuardrailVerifier("ok", lookup={"ok": judge}.get).verify(outcome) + stale_answer_behind_a_redacted_turn = ProgramOutcome( + program_id="p", + model="smart", + prompt="task", + response_text="an earlier public answer", + cost=0.1, + messages=[{"role": "user", "content": REDACTED_BY_LITELLM}], + ) + with pytest.raises(RuntimeError, match="redacted before verification"): + await GuardrailVerifier("ok", lookup={"ok": judge}.get).verify(stale_answer_behind_a_redacted_turn) + assert judge.seen == [] + + @pytest.mark.asyncio async def test_guardrail_verifier_hands_the_judge_the_transcript_and_the_feedback_payload(): passing, blocking = _Guardrail(blocks=False), _Guardrail(blocks=True)