diff --git a/builtin-skills/skills/zero-slop/scripts/slopscore.py b/builtin-skills/skills/zero-slop/scripts/slopscore.py index d31f98aa..9fa88124 100644 --- a/builtin-skills/skills/zero-slop/scripts/slopscore.py +++ b/builtin-skills/skills/zero-slop/scripts/slopscore.py @@ -33,6 +33,9 @@ from pathlib import Path DATA_DIR = Path(__file__).resolve().parent.parent / "data" SHAPE_SOLO_THRESHOLD = 0.62 # calibrated, see calibrate.py --shape +MAX_BATCH_FILES = 1_000 +MAX_BATCH_FILE_BYTES = 10 * 1024 * 1024 +MAX_BATCH_TOTAL_BYTES = 100 * 1024 * 1024 # Where personal voice profiles live โ€” outside the repo, since they are the @@ -1156,6 +1159,51 @@ def facts(text, _other=""): return out +LIMITATION_RX = re.compile( + r"\b(?:not|no|never|without|unmeasured|unknown|uncertain|unable|cannot|can't|" + r"didn't|doesn't|isn't|wasn't|weren't|hasn't|haven't|hadn't)\b", + re.I, +) +LIMITATION_STOP_WORDS = { + "a", "an", "and", "are", "as", "at", "be", "been", "but", "by", "did", + "do", "does", "for", "from", "had", "has", "have", "he", "her", "his", + "i", "in", "is", "it", "its", "no", "not", "of", "on", "or", "our", + "she", "that", "the", "their", "they", "this", "to", "was", "we", "were", + "with", "without", "you", "never", "unable", "cannot", "unknown", "uncertain", +} +LIMITATION_CANON = { + "measure": "measure", "measured": "measure", "measuring": "measure", + "measurement": "measure", "measurements": "measure", + "track": "track", "tracked": "track", "tracking": "track", + "test": "test", "tested": "test", "testing": "test", + "verify": "verify", "verified": "verify", "verifying": "verify", + "verification": "verify", + "assess": "assess", "assessed": "assess", "assessing": "assess", + "assessment": "assess", +} + + +def limitation_claims(text): + """Conservative signatures for explicitly qualified or negative claims. + + These signatures are intentionally a backstop, not semantic equivalence. + If a rewrite substantially rephrases a limitation, the assistant must + compare it manually rather than silently accepting a possible reversal. + """ + out = set() + for sentence in sentences(text): + if not LIMITATION_RX.search(sentence): + continue + words = [] + for word in re.findall(r"[A-Za-z][A-Za-z'-]*", sentence.lower()): + canonical = LIMITATION_CANON.get(word, word) + if canonical not in LIMITATION_STOP_WORDS and len(canonical) > 1: + words.append(canonical) + if words: + out.add(" ".join(sorted(set(words)))) + return out + + # Interior states the author has to have supplied. The benchmark's one # fabrication was exactly this shape โ€” "by test day the real thing felt # familiar" โ€” and an entity check cannot see it, because no name or figure moved. @@ -1488,6 +1536,16 @@ def fidelity(before, after, adjudicated=None): kept_all = False if added: invented_any = True + before_limitations = limitation_claims(before) + after_limitations = limitation_claims(after) + kept_limitations = before_limitations & after_limitations + dropped_limitations = before_limitations - after_limitations + added_limitations = after_limitations - before_limitations + if before_limitations or after_limitations: + rows.append(("qualifier", kept_limitations, dropped_limitations, + added_limitations)) + kept_all = kept_all and not dropped_limitations + invented_any = invented_any or bool(added_limitations) if new_interior: rows.append(("feeling", set(), set(), new_interior)) invented_any = True @@ -1628,7 +1686,8 @@ def render_fidelity(before, after, adjudicated=None): if r["preserved"] and not r["invented"] else ("SOURCE CONTENT CHANGED" if not r["preserved"] else "") + (" ยท CONTENT INVENTED" if r["invented"] else "")), - " This checks figures, names, quotes, links, stated feelings, code,", + " This checks figures, names, quotes, links, explicit limitations,", + " stated feelings, code,", " front matter, tables, blockquotes, inline identifiers, paths, and headings.", " Your AI assistant still compares the full meaning because a changed claim", " or emphasis may use all the same names and numbers.", ""] @@ -1700,8 +1759,35 @@ def _text_files(root_arg): raise SystemExit(f"directory does not exist: {root}") if not root.is_dir(): raise SystemExit(f"expected a directory, got: {root}") - return sorted(p for p in root.rglob("*") if p.suffix.lower() in - (".md", ".txt", ".markdown") and p.is_file()) + root = root.resolve() + files = [] + total_bytes = 0 + for path in root.rglob("*"): + if path.suffix.lower() not in (".md", ".txt", ".markdown"): + continue + if path.is_symlink(): + raise SystemExit(f"symbolic links are not allowed in recursive input: {path}") + try: + resolved = path.resolve(strict=True) + resolved.relative_to(root) + except (OSError, ValueError) as exc: + raise SystemExit(f"input resolves outside the selected directory: {path}") from exc + if not resolved.is_file(): + continue + size = resolved.stat().st_size + if size > MAX_BATCH_FILE_BYTES: + raise SystemExit( + f"input file exceeds {MAX_BATCH_FILE_BYTES} bytes: {path}" + ) + files.append(resolved) + total_bytes += size + if len(files) > MAX_BATCH_FILES: + raise SystemExit(f"recursive input exceeds {MAX_BATCH_FILES} text files") + if total_bytes > MAX_BATCH_TOTAL_BYTES: + raise SystemExit( + f"recursive input exceeds {MAX_BATCH_TOTAL_BYTES} total bytes" + ) + return sorted(files) def main(): diff --git a/scripts/tests/build-builtin-skills-test.sh b/scripts/tests/build-builtin-skills-test.sh index 946e5315..26f210c5 100755 --- a/scripts/tests/build-builtin-skills-test.sh +++ b/scripts/tests/build-builtin-skills-test.sh @@ -168,6 +168,8 @@ for artifact in data["artifacts"]: PY ) +python3 "$REPO_ROOT/scripts/tests/test_zero_slop.py" + mini_source="$tmp/mini-source" mkdir -p "$mini_source" cp -R "$REPO_ROOT/builtin-skills/skills/exam-ready" "$mini_source/exam-ready" diff --git a/scripts/tests/test_zero_slop.py b/scripts/tests/test_zero_slop.py new file mode 100644 index 00000000..b0cdb74e --- /dev/null +++ b/scripts/tests/test_zero_slop.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Behavior and safety regression tests for the reviewed Zero Slop package.""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path +import subprocess +import sys +import tempfile +import time +import unittest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +SCRIPT = REPO_ROOT / "builtin-skills/skills/zero-slop/scripts/slopscore.py" +sys.dont_write_bytecode = True +SPEC = importlib.util.spec_from_file_location("zero_slop_scorer", SCRIPT) +assert SPEC and SPEC.loader +SCORER = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(SCORER) + + +class ZeroSlopTests(unittest.TestCase): + def run_cli(self, *args: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["python3", str(SCRIPT), *args], input=stdin, text=True, + capture_output=True, check=False, + ) + + def test_score_is_offline_json_in_zero_to_one_hundred_range(self) -> None: + result = self.run_cli("--json", "-", stdin="We are thrilled to announce a seamless pilot.") + self.assertEqual(0, result.returncode, result.stderr) + payload = json.loads(result.stdout) + self.assertGreaterEqual(payload["ai_likelihood"], 0) + self.assertLessEqual(payload["ai_likelihood"], 100) + self.assertTrue(payload["hits"]) + + def test_fidelity_preserves_facts_and_rejects_dropped_figure(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + before = root / "before.md" + after = root / "after.md" + before.write_text("On 12 March, Maya said \"keep it read-only.\" Retries fell 17%.", encoding="utf-8") + after.write_text("Retries fell 17%. On 12 March, Maya said \"keep it read-only.\"", encoding="utf-8") + self.assertEqual(0, self.run_cli("--fidelity", str(before), str(after)).returncode) + after.write_text("On 12 March, Maya said \"keep it read-only.\"", encoding="utf-8") + self.assertEqual(1, self.run_cli("--fidelity", str(before), str(after)).returncode) + + def test_fidelity_rejects_dropped_or_reversed_limitation(self) -> None: + before = "The pilot included 48 users. We did not measure retention." + self.assertFalse(SCORER.fidelity(before, "The pilot included 48 users.")["preserved"]) + reversed_claim = "The pilot included 48 users. We measured retention." + self.assertFalse(SCORER.fidelity(before, reversed_claim)["preserved"]) + + def test_recursive_input_rejects_file_symlink(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + outside = root.parent / f"{root.name}-private.md" + outside.write_text("private five word phrase must stay private", encoding="utf-8") + try: + (root / "outside.md").symlink_to(outside) + result = self.run_cli("--portfolio", str(root)) + self.assertNotEqual(0, result.returncode) + self.assertIn("symbolic links are not allowed", result.stderr) + self.assertNotIn("private five word phrase", result.stdout) + finally: + outside.unlink(missing_ok=True) + + def test_recursive_input_enforces_file_count_and_size_budgets(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "one.md").write_text("one", encoding="utf-8") + (root / "two.md").write_text("two", encoding="utf-8") + old_count, old_size = SCORER.MAX_BATCH_FILES, SCORER.MAX_BATCH_FILE_BYTES + try: + SCORER.MAX_BATCH_FILES = 1 + with self.assertRaisesRegex(SystemExit, "exceeds 1 text files"): + SCORER._text_files(root) + SCORER.MAX_BATCH_FILES = old_count + SCORER.MAX_BATCH_FILE_BYTES = 2 + with self.assertRaisesRegex(SystemExit, "exceeds 2 bytes"): + SCORER._text_files(root) + finally: + SCORER.MAX_BATCH_FILES = old_count + SCORER.MAX_BATCH_FILE_BYTES = old_size + + def test_all_reviewed_patterns_compile_and_batch_gate_exit_codes(self) -> None: + data = SCORER.load_patterns() + self.assertEqual(len(data["patterns"]), len(SCORER._pattern_plan(data))) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "draft.md").write_text( + "We are thrilled to announce a transformative seamless experience.", + encoding="utf-8", + ) + self.assertEqual(1, self.run_cli("--batch", str(root), "--gate", "0").returncode) + self.assertEqual(0, self.run_cli("--batch", str(root), "--gate", "100").returncode) + + def test_one_thousand_short_documents_finish_within_generous_budget(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for index in range(1_000): + (root / f"draft-{index:04d}.md").write_text( + "A direct sentence with concrete wording.", encoding="utf-8" + ) + started = time.monotonic() + result = self.run_cli("--batch", str(root), "--json", "--gate", "100") + elapsed = time.monotonic() - started + self.assertEqual(0, result.returncode, result.stderr) + self.assertEqual(1_000, json.loads(result.stdout)["documents"]) + self.assertLess(elapsed, 30, f"batch regression: {elapsed:.2f}s") + + +if __name__ == "__main__": + unittest.main()