diff --git a/strix/core/execution.py b/strix/core/execution.py index d5d94eb57..d40da8ec0 100644 --- a/strix/core/execution.py +++ b/strix/core/execution.py @@ -86,6 +86,68 @@ def _structured_provider_refusal(result: Any) -> str | None: return None +# Plain-text refusals: the model returns a normal text turn (no structured +# ``refusal`` content type) instead of calling a lifecycle tool. Without +# detection this is treated as ordinary output and retried up to 1000 times, +# creating false confidence in scan completeness and wasting budget (#1155). +# +# Greptile review: unanchored single-phrase matching (e.g. "i cannot" or +# "as an ai") falsely flagged ordinary tool-less turns like +# "I cannot reproduce the issue" or quoted page content. Tighten to +# multi-word, action-anchored markers and require co-occurrence for +# ambiguous phrases. +_STRONG_REFUSAL_PHRASES: frozenset[str] = frozenset( + { + "i cannot help", + "i cannot assist", + "i can't help", + "i can't assist", + "cannot help with", + "cannot assist", + "unable to help", + "unable to assist", + "i will not help", + "i will not assist", + "refuse to help", + "refuse to assist", + "i must decline", + "i have to decline", + "can't help with", + "won't help with", + } +) + + +def _extract_final_output_text(result: Any) -> str | None: + """Return the final output as stripped text if it is a non-empty string.""" + final = getattr(result, "final_output", None) + if isinstance(final, str) and final.strip(): + return final.strip() + return None + + +def _is_text_refusal(text: str) -> bool: + """Heuristic: does plain-text output look like a model safety refusal? + + Uses action-anchored strong phrases plus guarded co-occurrence checks + for ambiguous markers to avoid false positives on ordinary turns + (e.g. quoted page content or 'I cannot reproduce the issue'). + """ + lower = text.lower() + if any(phrase in lower for phrase in _STRONG_REFUSAL_PHRASES): + return True + # Ambiguous markers only count when paired with an inability signal. + if "i apologize" in lower and any(k in lower for k in ("cannot", "unable", "won't", "can't")): + return True + if "as an ai" in lower and any(k in lower for k in ("cannot", "unable", "not able", "won't")): + return True + if ("safety guidelines" in lower or "my safety" in lower) and any( + k in lower for k in ("cannot", "unable", "refuse", "not able", "won't") + ): + return True + return False + + def _run_config_model(run_config: RunConfig) -> str | None: return run_config.model if isinstance(run_config.model, str) else None @@ -555,6 +617,22 @@ async def _run_until_lifecycle( await coordinator.reset_recovery(agent_id) return result + # Plain-text refusal: the model returned refusal text instead of a tool + # call. Treat as a coverage gap and fail fast rather than nudging up to + # 1000 times and reporting the scan as comprehensive (#1155). + refusal_text = _extract_final_output_text(result) + if refusal_text is not None and _is_text_refusal(refusal_text): + preview = _final_output_preview(result) + logger.error( + "agent %s refused task (plain-text refusal); failing fast instead of " + "retrying without tool call: %s", + agent_id, + preview, + ) + await coordinator.set_status(agent_id, "failed", error=f"model refusal: {preview}") + await notify_parent_on_terminal(coordinator, agent_id, "failed") + return result + recoveries = await coordinator.record_recovery(agent_id) logger.warning( "agent %s ended a turn without a lifecycle tool call (interactive=%s); "