fix(starter): harden Zero Slop batch validation

Signed-off-by: XiaoSeS <87064762+XiaoSeS@users.noreply.github.com>

builtin-skills/skills/zero-slop/scripts/slopscore.py

@@ -33,6 +33,9 @@ from pathlib import Path
 
 DATA_DIR = Path(__file__).resolve().parent.parent / "data"
 SHAPE_SOLO_THRESHOLD = 0.62  # calibrated, see calibrate.py --shape
+MAX_BATCH_FILES = 1_000
+MAX_BATCH_FILE_BYTES = 10 * 1024 * 1024
+MAX_BATCH_TOTAL_BYTES = 100 * 1024 * 1024
 
 
 # Where personal voice profiles live — outside the repo, since they are the
@@ -1156,6 +1159,51 @@ def facts(text, _other=""):
     return out
 
 
+LIMITATION_RX = re.compile(
+    r"\b(?:not|no|never|without|unmeasured|unknown|uncertain|unable|cannot|can't|"
+    r"didn't|doesn't|isn't|wasn't|weren't|hasn't|haven't|hadn't)\b",
+    re.I,
+)
+LIMITATION_STOP_WORDS = {
+    "a", "an", "and", "are", "as", "at", "be", "been", "but", "by", "did",
+    "do", "does", "for", "from", "had", "has", "have", "he", "her", "his",
+    "i", "in", "is", "it", "its", "no", "not", "of", "on", "or", "our",
+    "she", "that", "the", "their", "they", "this", "to", "was", "we", "were",
+    "with", "without", "you", "never", "unable", "cannot", "unknown", "uncertain",
+}
+LIMITATION_CANON = {
+    "measure": "measure", "measured": "measure", "measuring": "measure",
+    "measurement": "measure", "measurements": "measure",
+    "track": "track", "tracked": "track", "tracking": "track",
+    "test": "test", "tested": "test", "testing": "test",
+    "verify": "verify", "verified": "verify", "verifying": "verify",
+    "verification": "verify",
+    "assess": "assess", "assessed": "assess", "assessing": "assess",
+    "assessment": "assess",
+}
+
+
+def limitation_claims(text):
+    """Conservative signatures for explicitly qualified or negative claims.
+
+    These signatures are intentionally a backstop, not semantic equivalence.
+    If a rewrite substantially rephrases a limitation, the assistant must
+    compare it manually rather than silently accepting a possible reversal.
+    """
+    out = set()
+    for sentence in sentences(text):
+        if not LIMITATION_RX.search(sentence):
+            continue
+        words = []
+        for word in re.findall(r"[A-Za-z][A-Za-z'-]*", sentence.lower()):
+            canonical = LIMITATION_CANON.get(word, word)
+            if canonical not in LIMITATION_STOP_WORDS and len(canonical) > 1:
+                words.append(canonical)
+        if words:
+            out.add(" ".join(sorted(set(words))))
+    return out
+
+
 # Interior states the author has to have supplied. The benchmark's one
 # fabrication was exactly this shape — "by test day the real thing felt
 # familiar" — and an entity check cannot see it, because no name or figure moved.
@@ -1488,6 +1536,16 @@ def fidelity(before, after, adjudicated=None):
             kept_all = False
         if added:
             invented_any = True
+    before_limitations = limitation_claims(before)
+    after_limitations = limitation_claims(after)
+    kept_limitations = before_limitations & after_limitations
+    dropped_limitations = before_limitations - after_limitations
+    added_limitations = after_limitations - before_limitations
+    if before_limitations or after_limitations:
+        rows.append(("qualifier", kept_limitations, dropped_limitations,
+                     added_limitations))
+        kept_all = kept_all and not dropped_limitations
+        invented_any = invented_any or bool(added_limitations)
     if new_interior:
         rows.append(("feeling", set(), set(), new_interior))
         invented_any = True
@@ -1628,7 +1686,8 @@ def render_fidelity(before, after, adjudicated=None):
                              if r["preserved"] and not r["invented"] else
                              ("SOURCE CONTENT CHANGED" if not r["preserved"] else "")
                              + (" · CONTENT INVENTED" if r["invented"] else "")),
-            "  This checks figures, names, quotes, links, stated feelings, code,",
+            "  This checks figures, names, quotes, links, explicit limitations,",
+            "  stated feelings, code,",
             "  front matter, tables, blockquotes, inline identifiers, paths, and headings.",
             "  Your AI assistant still compares the full meaning because a changed claim",
             "  or emphasis may use all the same names and numbers.", ""]
@@ -1700,8 +1759,35 @@ def _text_files(root_arg):
         raise SystemExit(f"directory does not exist: {root}")
     if not root.is_dir():
         raise SystemExit(f"expected a directory, got: {root}")
-    return sorted(p for p in root.rglob("*") if p.suffix.lower() in
-                  (".md", ".txt", ".markdown") and p.is_file())
+    root = root.resolve()
+    files = []
+    total_bytes = 0
+    for path in root.rglob("*"):
+        if path.suffix.lower() not in (".md", ".txt", ".markdown"):
+            continue
+        if path.is_symlink():
+            raise SystemExit(f"symbolic links are not allowed in recursive input: {path}")
+        try:
+            resolved = path.resolve(strict=True)
+            resolved.relative_to(root)
+        except (OSError, ValueError) as exc:
+            raise SystemExit(f"input resolves outside the selected directory: {path}") from exc
+        if not resolved.is_file():
+            continue
+        size = resolved.stat().st_size
+        if size > MAX_BATCH_FILE_BYTES:
+            raise SystemExit(
+                f"input file exceeds {MAX_BATCH_FILE_BYTES} bytes: {path}"
+            )
+        files.append(resolved)
+        total_bytes += size
+        if len(files) > MAX_BATCH_FILES:
+            raise SystemExit(f"recursive input exceeds {MAX_BATCH_FILES} text files")
+        if total_bytes > MAX_BATCH_TOTAL_BYTES:
+            raise SystemExit(
+                f"recursive input exceeds {MAX_BATCH_TOTAL_BYTES} total bytes"
+            )
+    return sorted(files)
 
 
 def main():

scripts/tests/build-builtin-skills-test.sh

@@ -168,6 +168,8 @@ for artifact in data["artifacts"]:
 PY
 )
 
+python3 "$REPO_ROOT/scripts/tests/test_zero_slop.py"
+
 mini_source="$tmp/mini-source"
 mkdir -p "$mini_source"
 cp -R "$REPO_ROOT/builtin-skills/skills/exam-ready" "$mini_source/exam-ready"

scripts/tests/test_zero_slop.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""Behavior and safety regression tests for the reviewed Zero Slop package."""
+
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import time
+import unittest
+
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = REPO_ROOT / "builtin-skills/skills/zero-slop/scripts/slopscore.py"
+sys.dont_write_bytecode = True
+SPEC = importlib.util.spec_from_file_location("zero_slop_scorer", SCRIPT)
+assert SPEC and SPEC.loader
+SCORER = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(SCORER)
+
+
+class ZeroSlopTests(unittest.TestCase):
+    def run_cli(self, *args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]:
+        return subprocess.run(
+            ["python3", str(SCRIPT), *args], input=stdin, text=True,
+            capture_output=True, check=False,
+        )
+
+    def test_score_is_offline_json_in_zero_to_one_hundred_range(self) -> None:
+        result = self.run_cli("--json", "-", stdin="We are thrilled to announce a seamless pilot.")
+        self.assertEqual(0, result.returncode, result.stderr)
+        payload = json.loads(result.stdout)
+        self.assertGreaterEqual(payload["ai_likelihood"], 0)
+        self.assertLessEqual(payload["ai_likelihood"], 100)
+        self.assertTrue(payload["hits"])
+
+    def test_fidelity_preserves_facts_and_rejects_dropped_figure(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            before = root / "before.md"
+            after = root / "after.md"
+            before.write_text("On 12 March, Maya said \"keep it read-only.\" Retries fell 17%.", encoding="utf-8")
+            after.write_text("Retries fell 17%. On 12 March, Maya said \"keep it read-only.\"", encoding="utf-8")
+            self.assertEqual(0, self.run_cli("--fidelity", str(before), str(after)).returncode)
+            after.write_text("On 12 March, Maya said \"keep it read-only.\"", encoding="utf-8")
+            self.assertEqual(1, self.run_cli("--fidelity", str(before), str(after)).returncode)
+
+    def test_fidelity_rejects_dropped_or_reversed_limitation(self) -> None:
+        before = "The pilot included 48 users. We did not measure retention."
+        self.assertFalse(SCORER.fidelity(before, "The pilot included 48 users.")["preserved"])
+        reversed_claim = "The pilot included 48 users. We measured retention."
+        self.assertFalse(SCORER.fidelity(before, reversed_claim)["preserved"])
+
+    def test_recursive_input_rejects_file_symlink(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            outside = root.parent / f"{root.name}-private.md"
+            outside.write_text("private five word phrase must stay private", encoding="utf-8")
+            try:
+                (root / "outside.md").symlink_to(outside)
+                result = self.run_cli("--portfolio", str(root))
+                self.assertNotEqual(0, result.returncode)
+                self.assertIn("symbolic links are not allowed", result.stderr)
+                self.assertNotIn("private five word phrase", result.stdout)
+            finally:
+                outside.unlink(missing_ok=True)
+
+    def test_recursive_input_enforces_file_count_and_size_budgets(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "one.md").write_text("one", encoding="utf-8")
+            (root / "two.md").write_text("two", encoding="utf-8")
+            old_count, old_size = SCORER.MAX_BATCH_FILES, SCORER.MAX_BATCH_FILE_BYTES
+            try:
+                SCORER.MAX_BATCH_FILES = 1
+                with self.assertRaisesRegex(SystemExit, "exceeds 1 text files"):
+                    SCORER._text_files(root)
+                SCORER.MAX_BATCH_FILES = old_count
+                SCORER.MAX_BATCH_FILE_BYTES = 2
+                with self.assertRaisesRegex(SystemExit, "exceeds 2 bytes"):
+                    SCORER._text_files(root)
+            finally:
+                SCORER.MAX_BATCH_FILES = old_count
+                SCORER.MAX_BATCH_FILE_BYTES = old_size
+
+    def test_all_reviewed_patterns_compile_and_batch_gate_exit_codes(self) -> None:
+        data = SCORER.load_patterns()
+        self.assertEqual(len(data["patterns"]), len(SCORER._pattern_plan(data)))
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            (root / "draft.md").write_text(
+                "We are thrilled to announce a transformative seamless experience.",
+                encoding="utf-8",
+            )
+            self.assertEqual(1, self.run_cli("--batch", str(root), "--gate", "0").returncode)
+            self.assertEqual(0, self.run_cli("--batch", str(root), "--gate", "100").returncode)
+
+    def test_one_thousand_short_documents_finish_within_generous_budget(self) -> None:
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            for index in range(1_000):
+                (root / f"draft-{index:04d}.md").write_text(
+                    "A direct sentence with concrete wording.", encoding="utf-8"
+                )
+            started = time.monotonic()
+            result = self.run_cli("--batch", str(root), "--json", "--gate", "100")
+            elapsed = time.monotonic() - started
+            self.assertEqual(0, result.returncode, result.stderr)
+            self.assertEqual(1_000, json.loads(result.stdout)["documents"])
+            self.assertLess(elapsed, 30, f"batch regression: {elapsed:.2f}s")
+
+
+if __name__ == "__main__":
+    unittest.main()