From 0d4939e0cf5bf9e58619ab0e49e474d400afaa97 Mon Sep 17 00:00:00 2001
From: Claude <noreply@anthropic.com>
Date: Mon, 24 Aug 2026 20:56:41 +0000
Subject: [PATCH] fix(ci): don't count a Tessl CLI error as a sub-threshold
 quality score
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The quality-review workflow's parse fallback emits 'unknown|0|0|0|ERROR'
whenever the tessl CLI dies (auth/quota/npm failure), and the verdict logic
only compared SCORE against the threshold — so a tool outage rendered as
0/100 'NEEDS WORK' and blocked the merge, indistinguishable from a genuinely
zero-quality skill. Both the v2.12.0 promotion PR (#985) and #984 hit this:
four skills scored an identical 0/100 with the whole review loop finishing
in ~8 seconds.

Now VSTATUS=ERROR renders as a 'TOOL ERROR (not scored)' row with the CLI's
actual output surfaced as a ::warning:: annotation and a report footer, and
does not set the blocking exit code. Genuine sub-threshold scores still
block. Verified with a mocked-tessl simulation: error -> non-blocking warn,
85/100 -> PASS, 40/100 -> blocking NEEDS WORK, all-error run -> job passes.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Qgc6RYXWJPr5oW9DHU7zR4
---
 .github/workflows/skill-quality-review.yml | 25 +++++++++++++++++++---
 1 file changed, 22 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/skill-quality-review.yml b/.github/workflows/skill-quality-review.yml
index ad28537d..04b11bc4 100644
--- a/.github/workflows/skill-quality-review.yml
+++ b/.github/workflows/skill-quality-review.yml
@@ -99,6 +99,7 @@ jobs:
           SKILLS='${{ needs.detect-skills.outputs.skills }}'
           REPORT_FILE=$(mktemp)
           OVERALL_EXIT=0
+          TESSL_ERRORS=0
           THRESHOLD=70
 
           echo "## 📊 Skill Quality Review (Tessl)" > "$REPORT_FILE"
@@ -141,8 +142,17 @@ jobs:
 
             IFS='|' read -r NAME SCORE DS CS VSTATUS SUGGESTIONS <<< "$PARSED"
 
-            # Determine verdict
-            if [ "$SCORE" -ge "$THRESHOLD" ]; then
+            # Determine verdict. VSTATUS=ERROR means the Tessl CLI or the JSON
+            # parse failed before producing a review — that is a tool outage,
+            # not a genuine 0-score skill, so it must not block the merge
+            # (a real CLI failure scores every skill an identical 0/100; see
+            # the false-negative triage on the v2.12.0 promotion PR #985).
+            if [ "$VSTATUS" = "ERROR" ]; then
+              ICON="⚙️"
+              VERDICT="TOOL ERROR (not scored)"
+              TESSL_ERRORS=$((TESSL_ERRORS + 1))
+              echo "::warning::Tessl could not score $skill_dir — CLI output (truncated): $(echo "$JSON_OUT" | head -c 300)"
+            elif [ "$SCORE" -ge "$THRESHOLD" ]; then
               ICON="✅"
               VERDICT="PASS"
             else
@@ -151,7 +161,11 @@ jobs:
               OVERALL_EXIT=1
             fi
 
-            echo "| \`$skill_dir\` | **${SCORE}/100** ${ICON} | ${DS}% | ${CS}% | ${VERDICT} |" >> "$REPORT_FILE"
+            if [ "$VSTATUS" = "ERROR" ]; then
+              echo "| \`$skill_dir\` | ${ICON} not scored | — | — | ${VERDICT} |" >> "$REPORT_FILE"
+            else
+              echo "| \`$skill_dir\` | **${SCORE}/100** ${ICON} | ${DS}% | ${CS}% | ${VERDICT} |" >> "$REPORT_FILE"
+            fi
 
             # Add suggestions as details
             SUGG_COUNT=$(echo "$SUGGESTIONS" | python3 -c "import sys,json; print(len(json.loads(sys.stdin.readline())))" 2>/dev/null || echo "0")
@@ -184,6 +198,11 @@ jobs:
           echo "" >> "$REPORT_FILE"
           echo "_Threshold: ${THRESHOLD}/100 — skills below this score need improvement before merge._" >> "$REPORT_FILE"
 
+          if [ "$TESSL_ERRORS" -gt 0 ]; then
+            echo "" >> "$REPORT_FILE"
+            echo "> ⚙️ ${TESSL_ERRORS} skill(s) could not be scored — the Tessl CLI errored before producing a review. Reported as a warning, not a merge blocker; if this persists, check the Tessl CLI install/auth in this workflow." >> "$REPORT_FILE"
+          fi
+
           echo "report_file=$REPORT_FILE" >> "$GITHUB_OUTPUT"
           echo "exit_code=$OVERALL_EXIT" >> "$GITHUB_OUTPUT"