diff --git a/.github/workflows/ci-quality-gate.yml b/.github/workflows/ci-quality-gate.yml index 8e0b9fcd..0e4cb119 100644 --- a/.github/workflows/ci-quality-gate.yml +++ b/.github/workflows/ci-quality-gate.yml @@ -102,6 +102,13 @@ jobs: run: | python3 scripts/check_paths.py --all + # Every other gate reads frontmatter with a regex, so malformed YAML used + # to pass CI while Claude Code silently loaded the skill with no + # description. G10 parses it properly. + - name: Frontmatter YAML validator (gate G10 — blocking) + run: | + python3 scripts/check_frontmatter.py --all + - name: Dual-publish drift guard (gate G4 — blocking) run: | python3 scripts/check_dual_publish.py diff --git a/scripts/check_frontmatter.py b/scripts/check_frontmatter.py new file mode 100644 index 00000000..852a43f0 --- /dev/null +++ b/scripts/check_frontmatter.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 +"""check_frontmatter.py — YAML frontmatter validator for skills, agents, and commands. + +Every other gate in this repo extracts frontmatter with a regex or a line scan +(see generate-docs.py and sync-codex-skills.py), so a block that is not valid +YAML sails through CI unnoticed. Claude Code parses it properly: when the YAML +is malformed it loads the body with EMPTY metadata, which silently removes the +skill's `description` from the skill listing and makes it invisible to +model invocation. Agents, where `name` and `description` are required, may fail +to load entirely. + +This gate parses each block with yaml.safe_load and enforces the fields Claude +Code actually reads. + +Errors (exit 1): + - frontmatter block present but not parseable, or not a YAML mapping + - missing `description` (skills/commands) or `name`/`description` (agents) + - an agent `name` containing ':' (rejected by Claude Code >= 2.1.218) + - no frontmatter block at all in a file that should have one + +Warnings (exit 0 unless --strict): + - keys outside the current Claude Code frontmatter spec + - combined description + when_to_use over the 1536-char skill-listing cap + +Exit codes: 0 = clean, 1 = at least one error (or a warning under --strict). +Intended as CI gate G10. + +Usage: + python3 scripts/check_frontmatter.py --all # scan canonical dirs repo-wide + python3 scripts/check_frontmatter.py FILE [FILE ...] # scan specific files + python3 scripts/check_frontmatter.py --all --json # machine-readable output + python3 scripts/check_frontmatter.py --all --strict # warnings become errors +""" + +import argparse +import fnmatch +import json +import os +import re +import sys + +try: + import yaml +except ImportError: # pragma: no cover - CI installs yamllint, which vendors PyYAML + sys.stderr.write( + "check_frontmatter.py needs PyYAML (pip install pyyaml). " + "Parsing frontmatter with a regex is what this gate exists to prevent.\n" + ) + sys.exit(2) + +EXCLUDED_DIRS = { + ".git", ".codex", ".gemini", ".hermes", ".vibe", "docs", "audit", + "node_modules", ".github", ".claude-plugin", +} + +FRONTMATTER_RE = re.compile(r"^---\r?\n(.*?)\r?\n---\r?\n", re.DOTALL) + +# Files that legitimately carry no frontmatter: documentation that happens to +# live inside a canonical dir, and the skill-tester's deliberate bare fixture. +DOC_BASENAMES = {"README.md", "CLAUDE.md", "TEMPLATE.md"} +NO_FRONTMATTER_OK = ("*/assets/sample-skill/SKILL.md",) + +# Claude Code skill frontmatter. Commands are skills, so they share this set. +# https://code.claude.com/docs/en/skills#frontmatter-reference +SKILL_SPEC_KEYS = { + "name", "description", "when_to_use", "argument-hint", "arguments", + "disable-model-invocation", "user-invocable", "allowed-tools", + "disallowed-tools", "model", "effort", "context", "agent", "background", + "hooks", "paths", "shell", +} + +# Claude Code subagent frontmatter. +# https://code.claude.com/docs/en/sub-agents#write-subagent-files +AGENT_SPEC_KEYS = { + "name", "description", "tools", "disallowedTools", "model", + "permissionMode", "maxTurns", "skills", "mcpServers", "hooks", "memory", + "background", "effort", "isolation", "color", "initialPrompt", +} + +# Pre-existing repo conventions that no runtime reads. Reported separately from +# genuine typos so the migration in the v5 plan has a worklist rather than noise. +KNOWN_REPO_EXTRAS = { + "license", "metadata", "version", "author", "tags", "compatible_tools", + "triggers", "command", "domain", "title", "emoji", "vibe", "tier", + "category", "dependencies", "not_for", "agents", "source", "attribution", +} + +# Combined description + when_to_use budget per entry in the skill listing. +DESC_CAP = 1536 + + +def kind_for(rel_path): + """skill | agent | command, based on where the file lives.""" + if os.path.basename(rel_path) == "SKILL.md": + return "skill" + parts = rel_path.split(os.sep) + if "agents" in parts: + return "agent" + return "command" + + +def no_frontmatter_ok(rel_path): + if os.path.basename(rel_path) in DOC_BASENAMES: + return True + posix = rel_path.replace(os.sep, "/") + return any(fnmatch.fnmatch(posix, pat) for pat in NO_FRONTMATTER_OK) + + +def check_file(path, repo_root): + """Return (errors, warnings, offspec_keys) for one file.""" + rel = os.path.relpath(path, repo_root) + kind = kind_for(rel) + errors, warnings, offspec = [], [], [] + + text = open(path, encoding="utf-8", errors="replace").read() + match = FRONTMATTER_RE.match(text) + if not match: + if not no_frontmatter_ok(rel): + errors.append("no YAML frontmatter block") + return errors, warnings, offspec + + try: + data = yaml.safe_load(match.group(1)) + except yaml.YAMLError as exc: + detail = str(exc).replace("\n", " ") + errors.append(f"frontmatter is not valid YAML: {detail}") + return errors, warnings, offspec + + if not isinstance(data, dict): + errors.append(f"frontmatter is not a YAML mapping (parsed as {type(data).__name__})") + return errors, warnings, offspec + + if not str(data.get("description") or "").strip(): + errors.append("missing `description` — Claude Code has nothing to match the skill against") + + if kind == "agent": + name = str(data.get("name") or "").strip() + if not name: + errors.append("missing `name` — required for subagents") + elif ":" in name: + errors.append(f"`name: {name}` contains ':', which Claude Code >= 2.1.218 refuses to load") + + spec = AGENT_SPEC_KEYS if kind == "agent" else SKILL_SPEC_KEYS + for key in data: + if key in spec: + continue + if key in KNOWN_REPO_EXTRAS: + offspec.append(key) + else: + warnings.append(f"unrecognized key `{key}` (not in the {kind} frontmatter spec)") + + listing = len(str(data.get("description") or "")) + len(str(data.get("when_to_use") or "")) + if kind != "agent" and listing > DESC_CAP: + warnings.append( + f"description + when_to_use is {listing} chars; the skill listing truncates at {DESC_CAP}" + ) + + return errors, warnings, offspec + + +def collect_canonical(repo_root): + """All SKILL.md + *.md under any agents/ or commands/ dir, excluding sync/doc trees.""" + targets = [] + for dirpath, dirnames, filenames in os.walk(repo_root): + dirnames[:] = [d for d in dirnames if d not in EXCLUDED_DIRS] + parts = os.path.relpath(dirpath, repo_root).split(os.sep) + in_canonical_dir = "agents" in parts or "commands" in parts + for fn in filenames: + if fn == "SKILL.md" or (in_canonical_dir and fn.endswith(".md")): + targets.append(os.path.join(dirpath, fn)) + return sorted(targets) + + +def main(): + ap = argparse.ArgumentParser( + description="Validate YAML frontmatter on SKILL.md / agents / commands files." + ) + ap.add_argument("files", nargs="*", help="Specific markdown files to scan") + ap.add_argument("--all", action="store_true", + help="Scan all SKILL.md + agents/*.md + commands/*.md in the repo") + ap.add_argument("--json", action="store_true", help="Emit JSON instead of human-readable output") + ap.add_argument("--strict", action="store_true", help="Treat warnings as errors") + ap.add_argument("--root", default=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + help="Repo root (default: parent of this script)") + args = ap.parse_args() + + repo_root = os.path.abspath(args.root) + if args.all: + targets = collect_canonical(repo_root) + elif args.files: + targets = [os.path.abspath(f) for f in args.files] + else: + ap.print_help() + return 0 + + errors, warnings = {}, {} + offspec_counts = {} + for path in targets: + errs, warns, offspec = check_file(path, repo_root) + rel = os.path.relpath(path, repo_root) + if errs: + errors[rel] = errs + if warns: + warnings[rel] = warns + for key in offspec: + offspec_counts[key] = offspec_counts.get(key, 0) + 1 + + n_err = sum(len(v) for v in errors.values()) + n_warn = sum(len(v) for v in warnings.values()) + + if args.json: + print(json.dumps({ + "files_scanned": len(targets), + "errors": errors, + "warnings": warnings, + "total_errors": n_err, + "total_warnings": n_warn, + "off_spec_key_counts": dict(sorted(offspec_counts.items(), + key=lambda kv: -kv[1])), + }, indent=2)) + else: + for rel in sorted(errors): + print(f"{rel}:") + for msg in errors[rel]: + print(f" ERROR: {msg}") + for rel in sorted(warnings): + print(f"{rel}:") + for msg in warnings[rel]: + print(f" WARN: {msg}") + if offspec_counts: + print("\nOff-spec keys still in use (no runtime reads these):") + for key, count in sorted(offspec_counts.items(), key=lambda kv: -kv[1]): + print(f" {count:4d} {key}") + print(f"\nScanned {len(targets)} files; {n_err} errors, {n_warn} warnings.") + + if n_err: + return 1 + return 1 if (args.strict and n_warn) else 0 + + +if __name__ == "__main__": + sys.exit(main())