fix(oracle_router): never verify a program that has no answer yet

- Feedback (the endpoint, or program_done on a failed request) that arrives
  before any response of the program was observed no longer completes it:
  the completion is deferred on the binding and applied by the next observed
  response. Before, a harness could complete a program whose answer was still
  streaming; llm_as_a_judge skipped the empty answer without a verdict and
  the program scored 1.0. The feedback reply says `deferred: true`.
- The guardrail verifier fails the verification of a program whose final
  response has no text instead of reading a judge's skip as a pass.
- schema.d.ts regenerated for the new response field.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01HeiPj7YeRpKatjj2J7XhJ3

litellm/proxy/management_endpoints/oracle_router_endpoints.py

@@ -70,7 +70,9 @@ async def submit_oracle_router_feedback(
     """Complete a program: its model slot is released now, its verifier runs in the background.
 
     Programs are private to the key that started them; an admin may complete any key's program. 404 when
-    the caller holds no program with that id, which also covers programs completed earlier.
+    the caller holds no program with that id, which also covers programs completed earlier. A program none
+    of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed
+    response completes it with this feedback.
     """
     routers: Final = _oracle_routers()
     if not routers:
@@ -87,6 +89,7 @@ async def submit_oracle_router_feedback(
         router_name=holder.router_name,
         model=binding.model,
         pending_verifications=holder.pending_verifications,
+        deferred=binding.requests == 0,
     )
 
 

litellm/router_strategy/oracle_router/README.md

@@ -48,8 +48,13 @@ never reaches the decision maker.
 
 Either mark the last request with `metadata.program_done: true` (plus `program_score` for the reported
 verifier), or call `POST /oracle_router/feedback {"program_id": ..., "score": ..., "cost": ...}` from
-the harness once the task is graded. `GET /oracle_router/state` (admin) shows bindings, decision-maker
-state and recent feedback. The response header `x-litellm-oracle-router-model` names the bound model.
+the harness once the task is graded. Feedback that arrives before any response of the program was
+observed (it raced the last response's logging, or every request failed) is deferred: the reply says
+`deferred: true` and the next observed response completes the program with it, so nothing is ever
+verified or learned without an answer. A guardrail verifier also fails the verification of a program
+whose final response has no text, rather than reading a judge's skip as a pass. `GET /oracle_router/state`
+(admin) shows bindings, decision-maker state and recent feedback. The response header
+`x-litellm-oracle-router-model` names the bound model.
 
 ## Security
 

litellm/router_strategy/oracle_router/oracle_router.py

@@ -61,6 +61,15 @@ class ProgramKey(NamedTuple):
     program_id: str
 
 
+@dataclass(frozen=True, slots=True)
+class DeferredCompletion:
+    """A completion that arrived before any response of the program was observed; the next one applies it."""
+
+    score: float | None
+    cost: float | None
+    payload: Mapping[str, object]
+
+
 @dataclass(frozen=True, slots=True)
 class ProgramBinding:
     """One row of the lookup table: program id to model, plus what the proxy has seen of the program."""
@@ -74,6 +83,7 @@ class ProgramBinding:
     cost: float = 0.0
     last_response_text: str = ""
     last_messages: Sequence[AllMessageValues] = ()
+    deferred: DeferredCompletion | None = None
 
 
 @dataclass(frozen=True, slots=True)
@@ -304,6 +314,9 @@ class OracleRouter:
             last_response_text=response_text or binding.last_response_text,
             last_messages=transcript,
         )
+        deferred: Final = binding.deferred
+        if deferred is not None:  # the harness reported before this response was observed
+            self.complete(program_id, score=deferred.score, cost=deferred.cost, payload=deferred.payload, owner=owner)
 
     def complete(
         self,
@@ -313,10 +326,20 @@ class OracleRouter:
         payload: Mapping[str, object] = _EMPTY,
         owner: str | None = None,
     ) -> asyncio.Task[float] | None:
-        """Release the program now and verify it in the background; the task resolves to the verified score."""
-        binding: Final = self._bindings.pop(ProgramKey(owner, program_id), None)
+        """Release the program now and verify it in the background; the task resolves to the verified score.
+
+        A program none of whose requests has been observed yet (feedback that raced the last response's
+        logging, or a program whose only request failed) keeps its slot and is completed, with these
+        arguments, by its next observed response: there is no answer to verify yet, so nothing is learned.
+        """
+        key: Final = ProgramKey(owner, program_id)
+        binding: Final = self._bindings.get(key)
         if binding is None:
             return None
+        if binding.requests == 0:
+            self._bindings[key] = replace(binding, deferred=DeferredCompletion(score=score, cost=cost, payload=payload))
+            return None
+        del self._bindings[key]
         outcome: Final = ProgramOutcome(
             program_id=program_id,
             model=binding.model,

litellm/router_strategy/oracle_router/verifier.py

@@ -170,6 +170,9 @@ class GuardrailVerifier:
         guardrail: Final = self._lookup(self._name)
         if guardrail is None:
             raise LookupError(f"guardrail {self._name!r} is not initialized on this proxy")
+        if not outcome.response_text.strip():
+            # A judge skips an empty answer without recording a verdict; that must not read as a pass.
+            raise RuntimeError(f"guardrail {self._name!r}: the program has no response text to verify")
         transcript: Final = list(outcome.messages) or [{"role": "user", "content": outcome.prompt}]
         inputs: Final = GenericGuardrailAPIInputs(
             texts=[outcome.response_text], model=outcome.model, structured_messages=transcript

litellm/types/management_endpoints/oracle_router_endpoints.py

@@ -21,6 +21,10 @@ class OracleRouterFeedbackResponse(BaseModel):
     router_name: str
     model: str
     pending_verifications: int
+    deferred: bool = Field(
+        default=False,
+        description="No response of the program has been observed yet; its next one completes it with this feedback",
+    )
 
 
 class OracleRouterStateResponse(BaseModel):

tests/unit/proxy/management_endpoints/test_oracle_router_endpoints.py

@@ -72,6 +72,7 @@ async def test_feedback_completes_the_program_and_the_router_learns(oracle):
         "router_name": "oracle",
         "model": "smart",
         "pending_verifications": 1,
+        "deferred": False,
     }
     assert oracle.binding("task-1", "owner-hash") is None
     await oracle.drain()
@@ -81,6 +82,7 @@ async def test_feedback_completes_the_program_and_the_router_learns(oracle):
 @pytest.mark.asyncio
 async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it(oracle):
     await _bind(oracle, "task-1")
+    oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash")
     with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
         with pytest.raises(HTTPException) as unseen:
             await submit_oracle_router_feedback(OracleRouterFeedbackRequest(program_id="task-1", score=1.0), STRANGER)
@@ -96,6 +98,7 @@ async def test_another_key_does_not_see_the_program_but_an_admin_may_complete_it
 async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across_keys(oracle):
     await _bind(oracle, "task-1", "owner-hash")
     await _bind(oracle, "task-1", "other-hash")
+    oracle.observe_request("task-1", cost=0.2, response_text="done", owner="other-hash")
     with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
         response = await submit_oracle_router_feedback(
             OracleRouterFeedbackRequest(program_id="task-1", score=0.5), STRANGER
@@ -105,6 +108,21 @@ async def test_feedback_reaches_only_the_callers_program_when_ids_collide_across
     await oracle.drain()
 
 
+@pytest.mark.asyncio
+async def test_feedback_before_any_response_is_deferred_until_one_is_observed(oracle):
+    await _bind(oracle, "task-1")  # bound, but its first response has not been observed yet
+    with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):
+        response = await submit_oracle_router_feedback(
+            OracleRouterFeedbackRequest(program_id="task-1", score=1.0), OWNER
+        )
+    assert response.deferred is True and response.pending_verifications == 0
+    assert oracle.binding("task-1", "owner-hash") is not None and oracle.maker.updates == []
+    oracle.observe_request("task-1", cost=0.2, response_text="done", owner="owner-hash")
+    assert oracle.binding("task-1", "owner-hash") is None
+    await oracle.drain()
+    assert oracle.maker.updates == [("smart", 1.0)]
+
+
 @pytest.mark.asyncio
 async def test_feedback_for_an_unknown_program_or_without_routers_is_404(oracle):
     with patch("litellm.proxy.proxy_server.llm_router", _LLMRouter(oracle)):

tests/unit/router_strategy/oracle_router/test_hooks.py

@@ -100,11 +100,14 @@ async def test_a_non_finite_program_score_is_treated_as_no_score(bound):
 
 
 @pytest.mark.asyncio
-async def test_failure_event_does_not_add_spend_but_still_honours_program_done(bound):
+async def test_failure_event_adds_no_spend_and_program_done_on_it_waits_for_a_response(bound):
     router, maker, hook = bound
     await hook.async_log_failure_event(_kwargs(cost=0.5), None, None, None)
     assert router.binding("t1").cost == 0.0
     await hook.async_log_failure_event(_kwargs(program_done=True), None, None, None)
+    assert router.binding("t1") is not None and maker.updates == []  # no answer to verify yet: deferred
+    await hook.async_log_success_event(_kwargs(cost=0.1), _response("late answer"), None, None)
+    assert router.binding("t1") is None
     await router.drain()
     assert maker.updates == [("smart", 0.0)]
 

tests/unit/router_strategy/oracle_router/test_oracle_router.py

@@ -142,6 +142,7 @@ async def test_a_non_finite_verifier_score_is_a_failed_verification():
     await router.async_pre_routing_hook(
         model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
     )
+    router.observe_request("t", cost=0.1, response_text="answer")
     result = await router.complete("t")
     assert result != result and router.verifications_failed == 1 and maker.updates == []
     assert router.binding("t") is None  # the slot was released all the same
@@ -166,6 +167,7 @@ async def test_complete_releases_the_slot_now_and_learns_in_verification_order()
         )
     router.observe_request("t1", cost=0.4, response_text="patch")
     router.observe_request("t1", cost=0.2, response_text="tests pass")
+    router.observe_request("t2", cost=0.1, response_text="wrong")
     t1 = router.complete("t1", score=1.0)
     t2 = router.complete("t2", score=0.0)
     assert router.binding("t1") is None and router.binding("t2") is None
@@ -210,6 +212,7 @@ async def test_failed_verification_is_counted_and_does_not_update():
     await router.async_pre_routing_hook(
         model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
     )
+    router.observe_request("t", cost=0.1, response_text="answer")
     result = await router.complete("t")
     assert result != result and router.verifications_failed == 1 and maker.updates == []
 
@@ -253,11 +256,28 @@ async def test_the_same_program_id_under_another_key_is_another_program():
     assert router.programs_bound == 2 and maker.selections == 2
     assert router.binding("t1") is None  # no key: not the same program either
     assert [binding.api_key_hash for binding in router.bindings_named("t1")] == ["key-a", "key-b"]
+    router.observe_request("t1", cost=0.1, response_text="answer", owner="key-b")
     router.complete("t1", score=1.0, owner="key-b")
     assert router.binding("t1", "key-a") is not None and router.binding("t1", "key-b") is None
     await router.drain()
 
 
+@pytest.mark.asyncio
+async def test_completion_before_any_observed_response_is_deferred_to_the_next_one():
+    router, maker = _router()
+    await router.async_pre_routing_hook(
+        model="oracle", request_kwargs={"metadata": {"program_id": "t"}}, messages=_messages()
+    )
+    assert router.complete("t", score=1.0, cost=2.0, payload={"run": "r1"}) is None  # nothing answered yet
+    binding = router.binding("t")
+    assert binding is not None and binding.deferred is not None and router.programs_completed == 0
+    assert maker.updates == []
+    router.observe_request("t", cost=0.3, response_text="answer")  # the response the feedback raced
+    assert router.binding("t") is None and router.programs_completed == 1
+    await router.drain()
+    assert maker.updates == [("t", "smart", 1.0)]
+
+
 @pytest.mark.asyncio
 async def test_pre_routing_hook_replaces_caller_supplied_internal_keys():
     router, _ = _router()

tests/unit/router_strategy/oracle_router/test_verifier.py

@@ -104,6 +104,14 @@ async def test_guardrail_verifier_reads_the_recorded_verdict_before_the_exceptio
     assert await GuardrailVerifier("flagged", lookup=judge.get).verify(_outcome()) == 0.0
 
 
+@pytest.mark.asyncio
+async def test_a_program_without_response_text_is_a_failed_verification_not_a_pass():
+    judge = _RecordingGuardrail("success", name="judge")  # would skip an empty answer without a record
+    with pytest.raises(RuntimeError, match="no response text"):
+        await GuardrailVerifier("judge", lookup={"judge": judge}.get).verify(_outcome(answer="   "))
+    assert not hasattr(judge, "seen")  # the guardrail was never invoked
+
+
 @pytest.mark.asyncio
 async def test_feedback_payload_cannot_pre_seed_the_status_record_or_rename_the_program():
     judge = _RecordingGuardrail("guardrail_intervened", name="judge")  # on_failure: log -> no exception

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -11450,7 +11450,9 @@ export interface paths {
          * @description Complete a program: its model slot is released now, its verifier runs in the background.
          *
          *     Programs are private to the key that started them; an admin may complete any key's program. 404 when
-         *     the caller holds no program with that id, which also covers programs completed earlier.
+         *     the caller holds no program with that id, which also covers programs completed earlier. A program none
+         *     of whose responses has been observed yet is not completed: ``deferred`` is true and its next observed
+         *     response completes it with this feedback.
          */
         post: operations["submit_oracle_router_feedback_oracle_router_feedback_post"];
         delete?: never;
@@ -39415,6 +39417,12 @@ export interface components {
         };
         /** OracleRouterFeedbackResponse */
         OracleRouterFeedbackResponse: {
+            /**
+             * Deferred
+             * @description No response of the program has been observed yet; its next one completes it with this feedback
+             * @default false
+             */
+            deferred: boolean;
             /** Model */
             model: string;
             /** Pending Verifications */