claude-skills/scripts/audit_skills.py

#!/usr/bin/env python3
"""Run skill_review_checklist_runner on every SKILL.md in the repo + aggregate."""

import argparse
import json
import os
import subprocess
import sys

REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
RUNNER = os.path.join(
    REPO_ROOT,
    "engineering/write-a-skill/skills/write-a-skill/scripts/skill_review_checklist_runner.py"
)

EXCLUDE_PATTERNS = ("node_modules", "/.codex/", "/.gemini/", "/.cursor/", "/.cline/", "/site/", "/.git/", "/templates/", "assets/sample-skill")


def find_skills():
    skills = []
    for root, _, files in os.walk(REPO_ROOT):
        if any(p in root for p in EXCLUDE_PATTERNS):
            continue
        if "SKILL.md" in files:
            skills.append(root)
    return sorted(skills)


def audit_skill(skill_folder):
    """Run the runner; return parsed JSON result."""
    try:
        out = subprocess.run(
            ["python3", RUNNER, skill_folder, "--output", "json"],
            capture_output=True, text=True, timeout=10,
        )
        return json.loads(out.stdout)
    except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError) as e:
        return {"error": str(e), "folder": skill_folder, "passed": 0, "total": 6, "overall": "ERROR"}


def main():
    # Parse arguments BEFORE any work: `--help` must return instantly
    # instead of running the full ~30s repo-wide audit.
    parser = argparse.ArgumentParser(
        description="Run the write-a-skill review checklist on every SKILL.md "
                    "in the repo and print an aggregate report (~30s on the "
                    "full tree). Running with no arguments audits everything.")
    parser.parse_args()

    skills = find_skills()
    print(f"Auditing {len(skills)} skills...\n", file=sys.stderr)

    results = []
    for i, folder in enumerate(skills, 1):
        if i % 50 == 0:
            print(f"  ... {i}/{len(skills)}", file=sys.stderr)
        results.append(audit_skill(folder))

    # Aggregate
    pass_count = sum(1 for r in results if r.get("overall") == "PASS")
    warn_count = sum(1 for r in results if r.get("overall") == "WARN")
    fail_count = sum(1 for r in results if r.get("overall") == "FAIL")
    error_count = sum(1 for r in results if r.get("overall") == "ERROR")

    # Count failures by rule
    rule_failures = {}
    for r in results:
        for check in r.get("checks", []):
            if not check.get("pass"):
                rule = check.get("rule", "unknown")
                rule_failures[rule] = rule_failures.get(rule, 0) + 1

    # Top-10 worst (most failed checks)
    worst = sorted(
        results,
        key=lambda r: (r.get("passed", 0), -len(r.get("folder", ""))),
    )[:10]

    print("=" * 72)
    print("REPO-WIDE SKILL AUDIT (write-a-skill review checklist)")
    print("=" * 72)
    print(f"\nTotal SKILL.md audited: {len(results)}")
    print(f"  PASS  (6/6): {pass_count} ({100*pass_count//max(len(results),1)}%)")
    print(f"  WARN  (5/6): {warn_count}")
    print(f"  FAIL  (≤4/6): {fail_count}")
    print(f"  ERROR (couldn't parse): {error_count}")

    print("\n" + "-" * 72)
    print("FAILURES BY RULE")
    print("-" * 72)
    for rule, count in sorted(rule_failures.items(), key=lambda x: -x[1]):
        pct = 100 * count // max(len(results), 1)
        print(f"  {count:>4d} ({pct:>3d}%)  {rule}")

    print("\n" + "-" * 72)
    print("TOP-10 WORST OFFENDERS")
    print("-" * 72)
    for w in worst:
        folder = w.get("folder", "?").replace(REPO_ROOT + "/", "")
        passed = w.get("passed", 0)
        failed_rules = [c["rule"] for c in w.get("checks", []) if not c.get("pass")]
        print(f"\n  [{passed}/6] {folder}")
        for fr in failed_rules:
            print(f"    - {fr}")


if __name__ == "__main__":
    main()