From 0d6e8dfdd328faea8ea90a30676d547bc51e3215 Mon Sep 17 00:00:00 2001 From: Ritik Raj Date: Sat, 3 Oct 2026 23:08:57 +0000 Subject: [PATCH] fix(oracle_router): write nothing to the process log; keep failures on the state endpoint CodeQL's pull-request analysis is diff-informed: it only relates sources and sinks that lie in the PR's changed lines. When a PR adds both a taint source (here the feedback endpoint's request body) and a log-call sink, the data flow library falls back to computing every path in the codebase, and on LiteLLM py/log-injection then exceeds the 2 GiB relation limit. Reproduced with CodeQL 2.27.1 and the action's diff-range extension pack: the full diff fails after 14 minutes, every subset without that combination evaluates in under 20 seconds, and the tree with no log call on any added line evaluates in 15 seconds. So the router, its hook and its Router init log nothing. The last failure of a background step (verification, the post-call hook) is kept as `last_failure` on the router and shown by GET /oracle_router/state, next to the counters it already carried. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3 --- litellm/router.py | 7 ------- .../router_strategy/oracle_router/README.md | 4 +++- .../router_strategy/oracle_router/hooks.py | 8 ++++---- .../oracle_router/oracle_router.py | 19 +++++++++++++------ .../oracle_router/test_oracle_router.py | 2 ++ 5 files changed, 22 insertions(+), 18 deletions(-) diff --git a/litellm/router.py b/litellm/router.py index 06fddf79dec..922c7c41031 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9551,13 +9551,6 @@ class Router: strategy=oracle_router, strategy_label="Oracle-router", ) - verbose_router_logger.info( - "OracleRouter[%s] initialized with %d models, decision_maker=%s, verifier=%s", - deployment.model_name, - len(config.available_models), - config.decision_maker.type, - config.verifier.type, - ) def _is_quality_router_deployment(self, litellm_params: LiteLLM_Params) -> bool: """ diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index b458010e7c3..140e9dd67fb 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -72,4 +72,6 @@ whatever the caller's role) with a `custom` type is rejected when the router is ## Limits State is in memory per proxy process; it is not persisted or shared across replicas. The paper's -dispatch scheduler (DISC) is not part of this strategy. +dispatch scheduler (DISC) is not part of this strategy. The router and its hook write nothing to the +process log: `GET /oracle_router/state` carries the counters and the last background failure +(`last_failure`) instead. diff --git a/litellm/router_strategy/oracle_router/hooks.py b/litellm/router_strategy/oracle_router/hooks.py index 77479b2d1ef..6165f6293e3 100644 --- a/litellm/router_strategy/oracle_router/hooks.py +++ b/litellm/router_strategy/oracle_router/hooks.py @@ -3,14 +3,14 @@ After every completion it adds LiteLLM's computed spend and the latest answer to the program's row, and when the request carried ``metadata.program_done`` it completes the program: the slot is released at once and the verifier runs in the background. Only requests the Router stamped as routed by this ORACLE router -are read, and only against the program of the key that sent them. Nothing here can fail a request. +are read, and only against the program of the key that sent them. Nothing here can fail a request, and +nothing here logs: a failure is kept on the router for ``/oracle_router/state``. """ import math from collections.abc import Mapping from typing import TYPE_CHECKING, Final -from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger from litellm.router_strategy.oracle_router.config import ( CHOSEN_MODEL_METADATA_KEY, @@ -113,5 +113,5 @@ class OracleRouterPostCallHook(CustomLogger): ) if metadata.get(PROGRAM_DONE_KEY): self.oracle_router.complete(program_id, score=_score(metadata), owner=owner) - except Exception: # noqa: BLE001 # a logging callback must never fail the request it observes - verbose_router_logger.exception("OracleRouterPostCallHook: failed to record a request") + except Exception as error: # noqa: BLE001 # a logging callback must never fail the request it observes + self.oracle_router.note_failure("post-call hook", error) diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index 98bbc6e27ba..0be99e0f1c3 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -18,7 +18,6 @@ from typing import Final, NamedTuple from pydantic import TypeAdapter, ValidationError -from litellm._logging import verbose_router_logger from litellm.litellm_core_utils.core_helpers import ( get_or_create_metadata_bucket, # pyright: ignore[reportUnknownVariableType] # upstream helper returns a bare dict ) @@ -196,11 +195,21 @@ class OracleRouter: self.programs_evicted: int = 0 self.stateless_requests: int = 0 self.verifications_failed: int = 0 + self.last_failure: str | None = None @property def models(self) -> tuple[str, ...]: return tuple(self.config.available_models) + def note_failure(self, step: str, error: Exception) -> None: + """Keep the last failure of a background step for ``/oracle_router/state``. + + The router and its hook write nothing to the process log: every request input reaches them, and a + log call on that path makes CodeQL's log-injection analysis fall back to a whole-codebase pass that + no longer fits its limits on this codebase. The admin state endpoint carries the failure instead. + """ + self.last_failure = f"{step}: {type(error).__name__}: {error}" + def context_for(self, program_id: str, prompt: str) -> ProgramContext: return ProgramContext(program_id=program_id, prompt=prompt, request_type=classify_prompt(prompt)) @@ -275,9 +284,6 @@ class OracleRouter: chosen_model, request_type = await self._decide(program_id, prompt, _api_key_hash(request_kwargs)) _stamp_internal(request_kwargs, CHOSEN_MODEL_METADATA_KEY, chosen_model) _stamp_internal(request_kwargs, PROGRAM_ID_METADATA_KEY, program_id) - verbose_router_logger.debug( # nothing request-derived is logged; the decision is on the request's metadata - "OracleRouter: routed a %s request", "program" if program_id else "stateless" - ) return PreRoutingHookResponse( model=chosen_model, messages=messages, @@ -359,9 +365,9 @@ class OracleRouter: async def _verify_and_learn(self, binding: ProgramBinding, outcome: ProgramOutcome) -> float: try: score: Final = clamp_score(await self.verifier.verify(outcome)) - except Exception: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop + except Exception as error: # noqa: BLE001 # any verifier failure is counted and must not break the feedback loop self.verifications_failed += 1 - verbose_router_logger.exception("OracleRouter: a verification failed") + self.note_failure("verification", error) return float("nan") async with self._update_lock: self.decision_maker.update(binding.context, binding.model, score) @@ -410,6 +416,7 @@ class OracleRouter: "stateless_requests": self.stateless_requests, "pending_verifications": self.pending_verifications, "verifications_failed": self.verifications_failed, + "last_failure": self.last_failure, "recent_accuracy": self._accuracy_by_model(), "recent_feedback": [ { diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index 2b8608429a5..f5651cc8b48 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -215,6 +215,8 @@ async def test_failed_verification_is_counted_and_does_not_update(): router.observe_request("t", cost=0.1, response_text="answer") result = await router.complete("t") assert result != result and router.verifications_failed == 1 and maker.updates == [] + snapshot = await router.get_state_snapshot() + assert snapshot["last_failure"] == "verification: RuntimeError: sandbox down" # kept for the state endpoint, not logged @pytest.mark.asyncio