diff --git a/litellm/proxy/management_endpoints/oracle_router_endpoints.py b/litellm/proxy/management_endpoints/oracle_router_endpoints.py index a88bab1fe6e..63105933658 100644 --- a/litellm/proxy/management_endpoints/oracle_router_endpoints.py +++ b/litellm/proxy/management_endpoints/oracle_router_endpoints.py @@ -70,7 +70,9 @@ async def submit_oracle_router_feedback( """Complete a program: its model slot is released now, its verifier runs in the background. Programs are private to the key that started them; an admin may complete any key's program. 404 when - the caller holds no program with that id, which also covers programs completed earlier. + the caller holds no program with that id, which also covers programs completed earlier. A program none + of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed + response completes it with this feedback. """ routers: Final = _oracle_routers() if not routers: @@ -87,6 +89,7 @@ async def submit_oracle_router_feedback( router_name=holder.router_name, model=binding.model, pending_verifications=holder.pending_verifications, + deferred=binding.requests == 0, ) diff --git a/litellm/router_strategy/oracle_router/README.md b/litellm/router_strategy/oracle_router/README.md index 9d05c3ae81e..b458010e7c3 100644 --- a/litellm/router_strategy/oracle_router/README.md +++ b/litellm/router_strategy/oracle_router/README.md @@ -48,8 +48,13 @@ never reaches the decision maker. Either mark the last request with `metadata.program_done: true` (plus `program_score` for the reported verifier), or call `POST /oracle_router/feedback {"program_id": ..., "score": ..., "cost": ...}` from -the harness once the task is graded. `GET /oracle_router/state` (admin) shows bindings, decision-maker -state and recent feedback. The response header `x-litellm-oracle-router-model` names the bound model. +the harness once the task is graded. Feedback that arrives before any response of the program was +observed (it raced the last response's logging, or every request failed) is deferred: the reply says +`deferred: true` and the next observed response completes the program with it, so nothing is ever +verified or learned without an answer. A guardrail verifier also fails the verification of a program +whose final response has no text, rather than reading a judge's skip as a pass. `GET /oracle_router/state` +(admin) shows bindings, decision-maker state and recent feedback. The response header +`x-litellm-oracle-router-model` names the bound model. ## Security diff --git a/litellm/router_strategy/oracle_router/oracle_router.py b/litellm/router_strategy/oracle_router/oracle_router.py index eb3aff11667..98bbc6e27ba 100644 --- a/litellm/router_strategy/oracle_router/oracle_router.py +++ b/litellm/router_strategy/oracle_router/oracle_router.py @@ -61,6 +61,15 @@ class ProgramKey(NamedTuple): program_id: str +@dataclass(frozen=True, slots=True) +class DeferredCompletion: + """A completion that arrived before any response of the program was observed; the next one applies it.""" + + score: float | None + cost: float | None + payload: Mapping[str, object] + + @dataclass(frozen=True, slots=True) class ProgramBinding: """One row of the lookup table: program id to model, plus what the proxy has seen of the program.""" @@ -74,6 +83,7 @@ class ProgramBinding: cost: float = 0.0 last_response_text: str = "" last_messages: Sequence[AllMessageValues] = () + deferred: DeferredCompletion | None = None @dataclass(frozen=True, slots=True) @@ -304,6 +314,9 @@ class OracleRouter: last_response_text=response_text or binding.last_response_text, last_messages=transcript, ) + deferred: Final = binding.deferred + if deferred is not None: # the harness reported before this response was observed + self.complete(program_id, score=deferred.score, cost=deferred.cost, payload=deferred.payload, owner=owner) def complete( self, @@ -313,10 +326,20 @@ class OracleRouter: payload: Mapping[str, object] = _EMPTY, owner: str | None = None, ) -> asyncio.Task[float] | None: - """Release the program now and verify it in the background; the task resolves to the verified score.""" - binding: Final = self._bindings.pop(ProgramKey(owner, program_id), None) + """Release the program now and verify it in the background; the task resolves to the verified score. + + A program none of whose requests has been observed yet (feedback that raced the last response's + logging, or a program whose only request failed) keeps its slot and is completed, with these + arguments, by its next observed response: there is no answer to verify yet, so nothing is learned. + """ + key: Final = ProgramKey(owner, program_id) + binding: Final = self._bindings.get(key) if binding is None: return None + if binding.requests == 0: + self._bindings[key] = replace(binding, deferred=DeferredCompletion(score=score, cost=cost, payload=payload)) + return None + del self._bindings[key] outcome: Final = ProgramOutcome( program_id=program_id, model=binding.model, diff --git a/litellm/router_strategy/oracle_router/verifier.py b/litellm/router_strategy/oracle_router/verifier.py index e5c64fcf38b..5b202d90701 100644 --- a/litellm/router_strategy/oracle_router/verifier.py +++ b/litellm/router_strategy/oracle_router/verifier.py @@ -170,6 +170,9 @@ class GuardrailVerifier: guardrail: Final = self._lookup(self._name) if guardrail is None: raise LookupError(f"guardrail {self._name!r} is not initialized on this proxy") + if not outcome.response_text.strip(): + # A judge skips an empty answer without recording a verdict; that must not read as a pass. + raise RuntimeError(f"guardrail {self._name!r}: the program has no response text to verify") transcript: Final = list(outcome.messages) or [{"role": "user", "content": outcome.prompt}] inputs: Final = GenericGuardrailAPIInputs( texts=[outcome.response_text], model=outcome.model, structured_messages=transcript diff --git a/litellm/types/management_endpoints/oracle_router_endpoints.py b/litellm/types/management_endpoints/oracle_router_endpoints.py index cb5f63faf35..e400ee36100 100644 --- a/litellm/types/management_endpoints/oracle_router_endpoints.py +++ b/litellm/types/management_endpoints/oracle_router_endpoints.py @@ -21,6 +21,10 @@ class OracleRouterFeedbackResponse(BaseModel): router_name: str model: str pending_verifications: int + deferred: bool = Field( + default=False, + description="No response of the program has been observed yet; its next one completes it with this feedback", + ) class OracleRouterStateResponse(BaseModel): diff --git a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py index 7d0055dfebb..3c02f4ae8db 100644 --- a/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py @@ -72,6 +72,7 @@ async def test_feedback_completes_the_program_and_the_router_learns(oracle): "router_name": "oracle", "model": "smart", "pending_verifications": 1, + "deferred": False, } assert oracle.binding("task-1", "owner-hash") is None await oracle.drain() @@ -81,6 +82,7 @@ async def test_feedback_completes_the_program_and_the_router_learns(oracle): @pytest.mark.asyncio async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it(oracle): await _bind(oracle, "task-1") + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash") with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): with pytest.raises(HTTPException) as unseen: await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), STRANGER) @@ -96,6 +98,7 @@ async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across_keys(oracle): await _bind(oracle, "task-1", "owner-hash") await _bind(oracle, "task-1", "other-hash") + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="other-hash") with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): response = await submit_oracle_router_feedback( OracleRouterFeedbackRequest(program_id="task-1", score=0.5), STRANGER @@ -105,6 +108,21 @@ async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across await oracle.drain() +@pytest.mark.asyncio +async def test_feedback_before_any_response_is_deferred_until_one_is_observed(oracle): + await _bind(oracle, "task-1") # bound, but its first response has not been observed yet + with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): + response = await submit_oracle_router_feedback( + OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER + ) + assert response.deferred is True and response.pending_verifications == 0 + assert oracle.binding("task-1", "owner-hash") is not None and oracle.maker.updates == [] + oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash") + assert oracle.binding("task-1", "owner-hash") is None + await oracle.drain() + assert oracle.maker.updates == [("smart", 1.0)] + + @pytest.mark.asyncio async def test_feedback_for_an_unknown_program_or_without_routers_is_404(oracle): with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)): diff --git a/tests/unit/router_strategy/oracle_router/test_hooks.py b/tests/unit/router_strategy/oracle_router/test_hooks.py index d53b54f5e48..82e9710d755 100644 --- a/tests/unit/router_strategy/oracle_router/test_hooks.py +++ b/tests/unit/router_strategy/oracle_router/test_hooks.py @@ -100,11 +100,14 @@ async def test_a_non_finite_program_score_is_treated_as_no_score(bound): @pytest.mark.asyncio -async def test_failure_event_does_not_add_spend_but_still_honours_program_done(bound): +async def test_failure_event_adds_no_spend_and_program_done_on_it_waits_for_a_response(bound): router, maker, hook = bound await hook.async_log_failure_event(_kwargs(cost=0.5), None, None, None) assert router.binding("t1").cost == 0.0 await hook.async_log_failure_event(_kwargs(program_done=True), None, None, None) + assert router.binding("t1") is not None and maker.updates == [] # no answer to verify yet: deferred + await hook.async_log_success_event(_kwargs(cost=0.1), _response("late answer"), None, None) + assert router.binding("t1") is None await router.drain() assert maker.updates == [("smart", 0.0)] diff --git a/tests/unit/router_strategy/oracle_router/test_oracle_router.py b/tests/unit/router_strategy/oracle_router/test_oracle_router.py index 966bd905f79..2b8608429a5 100644 --- a/tests/unit/router_strategy/oracle_router/test_oracle_router.py +++ b/tests/unit/router_strategy/oracle_router/test_oracle_router.py @@ -142,6 +142,7 @@ async def test_a_non_finite_verifier_score_is_a_failed_verification(): await router.async_pre_routing_hook( model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() ) + router.observe_request("t", cost=0.1, response_text="answer") result = await router.complete("t") assert result != result and router.verifications_failed == 1 and maker.updates == [] assert router.binding("t") is None # the slot was released all the same @@ -166,6 +167,7 @@ async def test_complete_releases_the_slot_now_and_learns_in_verification_order() ) router.observe_request("t1", cost=0.4, response_text="patch") router.observe_request("t1", cost=0.2, response_text="tests pass") + router.observe_request("t2", cost=0.1, response_text="wrong") t1 = router.complete("t1", score=1.0) t2 = router.complete("t2", score=0.0) assert router.binding("t1") is None and router.binding("t2") is None @@ -210,6 +212,7 @@ async def test_failed_verification_is_counted_and_does_not_update(): await router.async_pre_routing_hook( model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() ) + router.observe_request("t", cost=0.1, response_text="answer") result = await router.complete("t") assert result != result and router.verifications_failed == 1 and maker.updates == [] @@ -253,11 +256,28 @@ async def test_the_same_program_id_under_another_key_is_another_program(): assert router.programs_bound == 2 and maker.selections == 2 assert router.binding("t1") is None # no key: not the same program either assert [binding.api_key_hash for binding in router.bindings_named("t1")] == ["key-a", "key-b"] + router.observe_request("t1", cost=0.1, response_text="answer", owner="key-b") router.complete("t1", score=1.0, owner="key-b") assert router.binding("t1", "key-a") is not None and router.binding("t1", "key-b") is None await router.drain() +@pytest.mark.asyncio +async def test_completion_before_any_observed_response_is_deferred_to_the_next_one(): + router, maker = _router() + await router.async_pre_routing_hook( + model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages() + ) + assert router.complete("t", score=1.0, cost=2.0, payload={"run": "r1"}) is None # nothing answered yet + binding = router.binding("t") + assert binding is not None and binding.deferred is not None and router.programs_completed == 0 + assert maker.updates == [] + router.observe_request("t", cost=0.3, response_text="answer") # the response the feedback raced + assert router.binding("t") is None and router.programs_completed == 1 + await router.drain() + assert maker.updates == [("t", "smart", 1.0)] + + @pytest.mark.asyncio async def test_pre_routing_hook_replaces_caller_supplied_internal_keys(): router, _ = _router() diff --git a/tests/unit/router_strategy/oracle_router/test_verifier.py b/tests/unit/router_strategy/oracle_router/test_verifier.py index a9219799faa..66164f6b914 100644 --- a/tests/unit/router_strategy/oracle_router/test_verifier.py +++ b/tests/unit/router_strategy/oracle_router/test_verifier.py @@ -104,6 +104,14 @@ async def test_guardrail_verifier_reads_the_recorded_verdict_before_the_exceptio assert await GuardrailVerifier("flagged", lookup=judge.get).verify(_outcome()) == 0.0 +@pytest.mark.asyncio +async def test_a_program_without_response_text_is_a_failed_verification_not_a_pass(): + judge = _RecordingGuardrail("success", name="judge") # would skip an empty answer without a record + with pytest.raises(RuntimeError, match="no response text"): + await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(_outcome(answer=" ")) + assert not hasattr(judge, "seen") # the guardrail was never invoked + + @pytest.mark.asyncio async def test_feedback_payload_cannot_pre_seed_the_status_record_or_rename_the_program(): judge = _RecordingGuardrail("guardrail_intervened", name="judge") # on_failure: log -> no exception diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 01230dd3137..18b19bb8925 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -11450,7 +11450,9 @@ export interface paths { * @description Complete a program: its model slot is released now, its verifier runs in the background. * * Programs are private to the key that started them; an admin may complete any key's program. 404 when - * the caller holds no program with that id, which also covers programs completed earlier. + * the caller holds no program with that id, which also covers programs completed earlier. A program none + * of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed + * response completes it with this feedback. */ post: operations["submit_oracle_router_feedback_oracle_router_feedback_post"]; delete?: never; @@ -39415,6 +39417,12 @@ export interface components { }; /** OracleRouterFeedbackResponse */ OracleRouterFeedbackResponse: { + /** + * Deferred + * @description No response of the program has been observed yet; its next one completes it with this feedback + * @default false + */ + deferred: boolean; /** Model */ model: string; /** Pending Verifications */