From d14f82cf2e09dbe270c0465ab0da2674d506ea65 Mon Sep 17 00:00:00 2001 From: harsshhh Date: Sat, 29 Aug 2026 12:28:48 +0530 Subject: [PATCH 1/2] fix(execution): fail fast on plain-text model refusal instead of retrying (#1155) When a specialist agent returns a safety refusal as plain text instead of a lifecycle tool call, the existing recovery loop treated it as ordinary output and retried up to 1000 times in non-interactive mode (max_turns) or 3 times interactive, wasting model budget and reporting false confidence in scan completeness. Structured provider refusals (type=refusal) were already handled in _run_cycle via ProviderRefusalError, but plain-text refusals were not. Add heuristic detection for text refusals (_TEXT_REFUSAL_PHRASES, _is_text_refusal, _extract_final_output_text) and handle in _run_until_lifecycle before burning recovery budget: set status to failed with 'model refusal: ', notify parent via notify_parent_on_terminal, and return immediately. Normal text-only turns still follow the existing nudge path. Fixes #1155 --- strix/core/execution.py | 57 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/strix/core/execution.py b/strix/core/execution.py index dfcd39fa2..d3aca07df 100644 --- a/strix/core/execution.py +++ b/strix/core/execution.py @@ -85,6 +85,47 @@ def _structured_provider_refusal(result: Any) -> str | None: return None +# Plain-text refusals: the model returns a normal text turn (no structured +# ``refusal`` content type) instead of calling a lifecycle tool. Without +# detection this is treated as ordinary output and retried up to 1000 times, +# creating false confidence in scan completeness and wasting budget (#1155). +_TEXT_REFUSAL_PHRASES: frozenset[str] = frozenset( + { + "i cannot", + "i can't", + "i am unable", + "i'm unable", + "i will not", + "cannot assist", + "unable to assist", + "i apologize", + "as an ai", + "my safety", + "safety guidelines", + "refuse to", + "not able to help", + "i must decline", + "i have to decline", + "can't help with", + "won't help with", + } +) + + +def _extract_final_output_text(result: Any) -> str | None: + """Return the final output as stripped text if it is a non-empty string.""" + final = getattr(result, "final_output", None) + if isinstance(final, str) and final.strip(): + return final.strip() + return None + + +def _is_text_refusal(text: str) -> bool: + """Heuristic: does plain-text output look like a model safety refusal?""" + lower = text.lower() + return any(phrase in lower for phrase in _TEXT_REFUSAL_PHRASES) + + def _run_config_model(run_config: RunConfig) -> str | None: return run_config.model if isinstance(run_config.model, str) else None @@ -516,6 +557,22 @@ async def _run_until_lifecycle( await coordinator.reset_recovery(agent_id) return result + # Plain-text refusal: the model returned refusal text instead of a tool + # call. Treat as a coverage gap and fail fast rather than nudging up to + # 1000 times and reporting the scan as comprehensive (#1155). + refusal_text = _extract_final_output_text(result) + if refusal_text is not None and _is_text_refusal(refusal_text): + preview = _final_output_preview(result) + logger.error( + "agent %s refused task (plain-text refusal); failing fast instead of " + "retrying without tool call: %s", + agent_id, + preview, + ) + await coordinator.set_status(agent_id, "failed", error=f"model refusal: {preview}") + await notify_parent_on_terminal(coordinator, agent_id, "failed") + return result + recoveries = await coordinator.record_recovery(agent_id) logger.warning( "agent %s ended a turn without a lifecycle tool call (interactive=%s); " From 6f0d7f6aeb621da7981aa64fc3f3002a517b37a6 Mon Sep 17 00:00:00 2001 From: harsshhh Date: Sat, 29 Aug 2026 12:47:08 +0530 Subject: [PATCH 2/2] fix(execution): tighten refusal heuristic to avoid false positives (Greptile P1) Address Greptile review: unanchored substring matching falsely flagged ordinary turns like 'I cannot reproduce the issue' or quoted page content containing 'as an AI'. Replace broad single-phrase set with action-anchored strong phrases (help/assist) and require co-occurrence for ambiguous markers (i apologize / as an ai / safety guidelines + inability). Verified: 'I cannot reproduce the issue' -> not refusal, quoted 'As an AI, we recommend...' -> not refusal, true refusals still detected, 61 tests pass. --- strix/core/execution.py | 49 +++++++++++++++++++++++++++++------------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/strix/core/execution.py b/strix/core/execution.py index d3aca07df..be4a92273 100644 --- a/strix/core/execution.py +++ b/strix/core/execution.py @@ -89,21 +89,26 @@ def _structured_provider_refusal(result: Any) -> str | None: # ``refusal`` content type) instead of calling a lifecycle tool. Without # detection this is treated as ordinary output and retried up to 1000 times, # creating false confidence in scan completeness and wasting budget (#1155). -_TEXT_REFUSAL_PHRASES: frozenset[str] = frozenset( +# +# Greptile review: unanchored single-phrase matching (e.g. "i cannot" or +# "as an ai") falsely flagged ordinary tool-less turns like +# "I cannot reproduce the issue" or quoted page content. Tighten to +# multi-word, action-anchored markers and require co-occurrence for +# ambiguous phrases. +_STRONG_REFUSAL_PHRASES: frozenset[str] = frozenset( { - "i cannot", - "i can't", - "i am unable", - "i'm unable", - "i will not", + "i cannot help", + "i cannot assist", + "i can't help", + "i can't assist", + "cannot help with", "cannot assist", + "unable to help", "unable to assist", - "i apologize", - "as an ai", - "my safety", - "safety guidelines", - "refuse to", - "not able to help", + "i will not help", + "i will not assist", + "refuse to help", + "refuse to assist", "i must decline", "i have to decline", "can't help with", @@ -121,9 +126,25 @@ def _extract_final_output_text(result: Any) -> str | None: def _is_text_refusal(text: str) -> bool: - """Heuristic: does plain-text output look like a model safety refusal?""" + """Heuristic: does plain-text output look like a model safety refusal? + + Uses action-anchored strong phrases plus guarded co-occurrence checks + for ambiguous markers to avoid false positives on ordinary turns + (e.g. quoted page content or 'I cannot reproduce the issue'). + """ lower = text.lower() - return any(phrase in lower for phrase in _TEXT_REFUSAL_PHRASES) + if any(phrase in lower for phrase in _STRONG_REFUSAL_PHRASES): + return True + # Ambiguous markers only count when paired with an inability signal. + if "i apologize" in lower and any(k in lower for k in ("cannot", "unable", "won't", "can't")): + return True + if "as an ai" in lower and any(k in lower for k in ("cannot", "unable", "not able", "won't")): + return True + if ("safety guidelines" in lower or "my safety" in lower) and any( + k in lower for k in ("cannot", "unable", "refuse", "not able", "won't") + ): + return True + return False def _run_config_model(run_config: RunConfig) -> str | None: