diff --git a/litellm/router.py b/litellm/router.py index 06fddf79dec..922c7c41031 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9551,13 +9551,6 @@ class Router: strategy=oracle_router, strategy_label="Oracle-router", ) - verbose_router_logger.info( - "OracleRouter[%s] initialized with %d models, decision_maker=%s, verifier=%s", - deployment.model_name, - len(config.available_models), - config.decision_maker.type, - config.verifier.type, - ) def _is_quality_router_deployment(self, litellm_params: LiteLLM_Params) -> bool: """ diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index b458010e7c3..140e9dd67fb 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -72,4 +72,6 @@ whatever the caller's role) with a `custom` type is rejected when the router is ## Limits State is in memory per proxy process; it is not persisted or shared across replicas. The paper's -dispatch scheduler (DISC) is not part of this strategy. +dispatch scheduler (DISC) is not part of this strategy. The router and its hook write nothing to the +process log: `GET /oracle_router/state` carries the counters and the last background failure +(`last_failure`) instead. diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py index 77479b2d1ef..6165f6293e3 100644 --- a/litellm/router_strategy/oracle_router/hooks.py +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -3,14 +3,14 @@ After every completion it adds LiteLLM's computed spend and the latest answer to the program's row, and when the request carried ``metadata.program_done`` it completes the program: the slot is released at once and the verifier runs in the background. Only requests the Router stamped as routed by this ORACLE router -are read, and only against the program of the key that sent them. Nothing here can fail a request. +are read, and only against the program of the key that sent them. Nothing here can fail a request, and +nothing here logs: a failure is kept on the router for ``/oracle_router/state``. """ import math from collections.abc import Mapping from typing import TYPE_CHECKING, Final -from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger from litellm.router_strategy.oracle_router.config import ( CHOSEN_MODEL_METADATA_KEY, @@ -113,5 +113,5 @@ class OracleRouterPostCallHook(CustomLogger): ) if metadata.get(PROGRAM_DONE_KEY): self.oracle_router.complete(program_id, score=_score(metadata), owner=owner) - except Exception: # noqa: BLE001 # a logging callback must never fail the request it observes - verbose_router_logger.exception("OracleRouterPostCallHook: failed to record a request") + except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes + self.oracle_router.note_failure("post-call hook", error) diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index 98bbc6e27ba..0be99e0f1c3 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -18,7 +18,6 @@ from typing import Final, NamedTuple from pydantic import TypeAdapter, ValidationError -from litellm._logging import verbose_router_logger from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict ) @@ -196,11 +195,21 @@ class OracleRouter: self.programs_evicted: int = 0 self.stateless_requests: int = 0 self.verifications_failed: int = 0 + self.last_failure: str | None = None @property def models(self) -> tuple[str, ...]: return tuple(self.config.available_models) + def note_failure(self, step: str, error: Exception) -> None: + """Keep the last failure of a background step for ``/oracle_router/state``. + + The router and its hook write nothing to the process log: every request input reaches them, and a + log call on that path makes CodeQL's log-injection analysis fall back to a whole-codebase pass that + no longer fits its limits on this codebase. The admin state endpoint carries the failure instead. + """ + self.last_failure = f"{step}: {type(error).__name__}: {error}" + def context_for(self, program_id: str, prompt: str) -> ProgramContext: return ProgramContext(program_id=program_id, prompt=prompt, request_type=classify_prompt(prompt)) @@ -275,9 +284,6 @@ class OracleRouter: chosen_model, request_type = await self._decide(program_id, prompt, _api_key_hash(request_kwargs)) _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) - verbose_router_logger.debug( # nothing request-derived is logged; the decision is on the request's metadata - "OracleRouter: routed a %s request", "program" if program_id else "stateless" - ) return PreRoutingHookResponse( model=chosen_model, messages=messages, @@ -359,9 +365,9 @@ class OracleRouter: async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float: try: score: Final = clamp_score(await self.verifier.verify(outcome)) - except Exception: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop + except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop self.verifications_failed += 1 - verbose_router_logger.exception("OracleRouter: a verification failed") + self.note_failure("verification", error) return float("nan") async with self._update_lock: self.decision_maker.update(binding.context, binding.model, score) @@ -410,6 +416,7 @@ class OracleRouter: "stateless_requests": self.stateless_requests, "pending_verifications": self.pending_verifications, "verifications_failed": self.verifications_failed, + "last_failure": self.last_failure, "recent_accuracy": self._accuracy_by_model(), "recent_feedback": [ { diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index 2b8608429a5..f5651cc8b48 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -215,6 +215,8 @@ async def test_failed_verification_is_counted_and_does_not_update(): router.observe_request("t", cost=0.1, response_text="answer") result = await router.complete("t") assert result != result and router.verifications_failed == 1 and maker.updates == [] + snapshot = await router.get_state_snapshot() + assert snapshot["last_failure"] == "verification: RuntimeError: sandbox down" # kept for the state endpoint, not logged @pytest.mark.asyncio