diff --git a/litellm/router_strategy/adaptive_router/signals.py b/litellm/router_strategy/adaptive_router/signals.py index bc67493bea6..edc3019fb4b 100644 --- a/litellm/router_strategy/adaptive_router/signals.py +++ b/litellm/router_strategy/adaptive_router/signals.py @@ -18,6 +18,7 @@ from typing import Any, Dict, List, Optional, Set from litellm.router_strategy.adaptive_router.config import ( LOOP_REPEAT_THRESHOLD, + MIN_TURNS_FOR_CLEAN_CREDIT, MISALIGNMENT_JACCARD_THRESHOLD, STAGNATION_JACCARD_NEAR_DUP, TOOL_CALL_HISTORY_MAX, @@ -80,6 +81,8 @@ class SessionState: pending_tool_calls: Dict[str, str] = field(default_factory=dict) turn_count: int = 0 + last_processed_turn: int = -1 + clean_credit_awarded: bool = False terminal_status: Optional[int] = None @@ -161,13 +164,15 @@ def _detect_satisfaction(curr_user: Optional[str]) -> bool: def _detect_failure(tool_results: List[Dict[str, Any]]) -> bool: - """Any tool result that's an error or empty content.""" + """Any tool result explicitly flagged as an error. + + We do NOT treat empty content as failure — many tools legitimately return + empty output (zero-result searches, silent bash commands, void writes) and + penalizing the model for those would corrupt the bandit posterior. + """ for r in tool_results: if r.get("is_error"): return True - content = r.get("content") - if content is None or content == "" or content == [] or content == {}: - return True return False @@ -238,7 +243,16 @@ def apply_turn(state: SessionState, turn: Turn) -> SignalDelta: if _detect_disengagement(turn.user_content): delta.disengagement = 1 if _detect_satisfaction(turn.user_content): - delta.satisfaction = 1 + # Gate: only award satisfaction credit once per session, and only + # after MIN_TURNS_FOR_CLEAN_CREDIT turns of context. Early "thanks" + # on turn 1-2 is noise, not a validated quality signal. + current_turn_index = state.turn_count + 1 + if ( + not state.clean_credit_awarded + and current_turn_index >= MIN_TURNS_FOR_CLEAN_CREDIT + ): + delta.satisfaction = 1 + state.clean_credit_awarded = True if _detect_failure(turn.tool_results): delta.failure = 1 if _detect_loop(state.tool_call_history, turn.tool_calls): @@ -268,5 +282,6 @@ def apply_turn(state: SessionState, turn: Turn) -> SignalDelta: state.terminal_status = turn.response_status state.turn_count += 1 + state.last_processed_turn = state.turn_count return delta diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_adaptive_router.py b/tests/test_litellm/router_strategy/adaptive_router/test_adaptive_router.py index 93f49398e2f..aed217cdc21 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_adaptive_router.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_adaptive_router.py @@ -124,6 +124,16 @@ def test_claim_or_check_owner_expired_owner_reclaims_for_new_model(monkeypatch): @pytest.mark.asyncio async def test_record_turn_pushes_to_queue(): r = _make_router() + # Prime with 2 prior turns so satisfaction gate (MIN_TURNS_FOR_CLEAN_CREDIT=3) + # is satisfied when the "thanks" turn arrives. + for _ in range(2): + await r.record_turn( + session_id="s1", + model_name="fast", + request_type=RequestType.GENERAL, + turn=Turn(user_content="hi", assistant_content="hello"), + ) + r.queue.add_session_state = AsyncMock() r.queue.add_state_delta = AsyncMock() @@ -143,6 +153,24 @@ async def test_record_turn_pushes_to_queue(): @pytest.mark.asyncio async def test_record_turn_satisfaction_increments_alpha(): r = _make_router() + # Prime with 2 prior turns to clear the MIN_TURNS_FOR_CLEAN_CREDIT gate. + # Use distinct content to avoid incidentally firing stagnation/misalignment. + priming_turns = [ + Turn( + user_content="alpha bravo charlie", assistant_content="delta echo foxtrot" + ), + Turn( + user_content="golf hotel india juliet", + assistant_content="kilo lima mike november", + ), + ] + for t in priming_turns: + await r.record_turn( + session_id="sX", + model_name="fast", + request_type=RequestType.GENERAL, + turn=t, + ) cell_before = r._cells[(RequestType.GENERAL, "fast")] turn = Turn(user_content="that worked, thanks!") await r.record_turn( @@ -224,7 +252,6 @@ async def test_load_state_from_db_handles_unknown_request_type(): assert r._cells[(RequestType.WRITING, "fast")] == cold or True - # ---- Session state eviction --------------------------------------------- @@ -267,4 +294,3 @@ def test_session_state_expiry_is_refreshed_on_access(): second_exp = r._session_states_expiry[("sess-A", "fast")] assert second_exp > first_exp - diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_e2e_adaptive_router.py b/tests/test_litellm/router_strategy/adaptive_router/test_e2e_adaptive_router.py index bb0e8df0445..9786832b4ae 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_e2e_adaptive_router.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_e2e_adaptive_router.py @@ -67,6 +67,24 @@ async def test_pick_record_flush_full_cycle(): chosen = await router.pick_model(RequestType.CODE_GENERATION) assert chosen in router.config.available_models + # Prime 2 prior turns (distinct content so no other signals fire) so the + # MIN_TURNS_FOR_CLEAN_CREDIT satisfaction gate is satisfied on turn 3. + priming = [ + Turn( + user_content="alpha bravo charlie", assistant_content="delta echo foxtrot" + ), + Turn( + user_content="golf hotel india juliet", + assistant_content="kilo lima mike november", + ), + ] + for t in priming: + await router.record_turn( + session_id="s1", + model_name=chosen, + request_type=RequestType.CODE_GENERATION, + turn=t, + ) await router.record_turn( session_id="s1", model_name=chosen, @@ -226,6 +244,15 @@ async def test_load_state_from_db_handles_unknown_request_type(): @pytest.mark.asyncio async def test_flush_isolates_writes_per_router_session_model(): router = _make_router() + # Prime 2 prior turns per session to clear the MIN_TURNS_FOR_CLEAN_CREDIT gate. + for sid, model in (("s1", "gpt-4o"), ("s2", "gpt-4o-mini")): + for _ in range(2): + await router.record_turn( + sid, + model, + RequestType.GENERAL, + Turn(user_content="hi", assistant_content="hello"), + ) await router.record_turn( "s1", "gpt-4o", RequestType.GENERAL, Turn(user_content="thanks!") ) @@ -248,6 +275,14 @@ async def test_repeated_flush_drains_queue_and_subsequent_flush_is_noop(): """Verifies the queue is fully drained on flush -- a second flush writes nothing.""" router = _make_router() chosen = await router.pick_model(RequestType.GENERAL) + # Prime 2 prior turns so satisfaction can fire on the third turn. + for _ in range(2): + await router.record_turn( + "drain-1", + chosen, + RequestType.GENERAL, + Turn(user_content="hi", assistant_content="hello"), + ) await router.record_turn( "drain-1", chosen, RequestType.GENERAL, Turn(user_content="thanks!") ) diff --git a/tests/test_litellm/router_strategy/adaptive_router/test_signals.py b/tests/test_litellm/router_strategy/adaptive_router/test_signals.py index bf09b1b16ff..2773c13a812 100644 --- a/tests/test_litellm/router_strategy/adaptive_router/test_signals.py +++ b/tests/test_litellm/router_strategy/adaptive_router/test_signals.py @@ -97,6 +97,74 @@ def test_mixed_failure_then_satisfaction(): assert state.satisfaction_count >= 1 +def test_satisfaction_gated_by_min_turns_for_clean_credit(): + """'thanks' on turn 1 is noise, not a validated quality signal.""" + state = SessionState( + session_id="s", router_name="r", model_name="m", classified_type="general" + ) + apply_turn(state, Turn(user_content="thanks!")) + assert state.satisfaction_count == 0 + assert state.clean_credit_awarded is False + assert state.last_processed_turn == 1 + + +def test_satisfaction_credit_awarded_once_per_session(): + """Even multiple satisfaction turns only award +1 alpha across the session.""" + state = SessionState( + session_id="s", router_name="r", model_name="m", classified_type="general" + ) + apply_turn(state, Turn(user_content="hi", assistant_content="hello")) + apply_turn(state, Turn(user_content="help me", assistant_content="sure")) + apply_turn(state, Turn(user_content="perfect, thanks")) + assert state.satisfaction_count == 1 + assert state.clean_credit_awarded is True + apply_turn(state, Turn(user_content="great, thank you")) + assert state.satisfaction_count == 1 + + +def test_empty_tool_content_does_not_fire_failure(): + """Zero-result searches / silent commands return empty but valid output.""" + state = SessionState( + session_id="s", router_name="r", model_name="m", classified_type="general" + ) + apply_turn( + state, + Turn( + tool_calls=[{"name": "grep", "arguments": {"q": "x"}}], + tool_results=[{"tool_call_id": "c1", "content": ""}], + ), + ) + apply_turn( + state, + Turn( + tool_calls=[{"name": "list", "arguments": {}}], + tool_results=[{"tool_call_id": "c2", "content": []}], + ), + ) + apply_turn( + state, + Turn( + tool_calls=[{"name": "noop", "arguments": {}}], + tool_results=[{"tool_call_id": "c3", "content": None}], + ), + ) + assert state.failure_count == 0 + + +def test_is_error_still_fires_failure(): + state = SessionState( + session_id="s", router_name="r", model_name="m", classified_type="general" + ) + apply_turn( + state, + Turn( + tool_calls=[{"name": "read", "arguments": {"p": "x"}}], + tool_results=[{"tool_call_id": "c1", "content": "boom", "is_error": True}], + ), + ) + assert state.failure_count == 1 + + def test_apply_turn_is_o1_does_not_grow_history_unbounded(): state = SessionState( session_id="s",