diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 7d0cc9de..247ef401 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ "homepage": "https://github.com/alirezarezvani/claude-skills", "repository": "https://github.com/alirezarezvani/claude-skills", "metadata": { - "description": "363 production-ready skills across 18 domains (engineering, engineering-core, marketing, product, c-level, compliance-os, project management, RA/QM, business growth, finance, productivity, marketing top-level, research, research-ops, business-operations, commercial, markdown-html, loop-library, plus standards). 662 Python tools, 746 reference guides, 103 agents (cs-* + personas), 118 slash commands across 89 marketplace plugins. v2.11.2 vendors engineering/skillopt-sleep — a verbatim copy of microsoft/SkillOpt's stdlib-only skillopt_sleep engine + Claude Code plugin surface, giving a local agent a nightly gated self-improvement cycle (read-only session harvest -> mine -> offline replay -> held-out-gated CLAUDE.md/SKILL.md edits -> staged for explicit /skillopt-sleep adopt). productivity/fable-goal (unreleased, post-v2.11.1) converts a rambling description of a desired outcome into one polished /goal prompt for a fresh autonomous session. v2.11.1 turns product-team and project-management into agent-harness domains: fork-orchestrators with deterministic goal routers, a Jira MCP snapshot bridge (Kanban flow metrics + Monte Carlo forecasting), a delegation-governance loop gate, a continuous-discovery cadence tracker, and an Opportunity Solution Tree linter, with /cs:pm and /cs:product command families. v2.10.3 completes the markdown-html domain with md-slides — slide-deck converter (arrow-key / Space / PgDn / Home/End / P keyboard navigation + presenter mode with split-view clock + speaker notes + next-slide preview + URL-hash deep linking like #3 for direct slide jumps + @media print page-per-slide for browser-native PDF export). Reuses md-document's markdown parser; vanilla JS only (no framework runtime); Prism.js opt-in via --syntax. Joins md-review (v2.10.2 code-review converter), md-document (v2.10.1 long-form converter), and the v2.10.0 foundation (orchestrator + design-system). Compatible with Claude Code, Codex CLI, Gemini CLI, Cursor, OpenClaw, Hermes Agent, Mistral Vibe, and 5 more coding agents.", + "description": "363 production-ready skills across 18 domains (engineering, engineering-core, marketing, product, c-level, compliance-os, project management, RA/QM, business growth, finance, productivity, marketing top-level, research, research-ops, business-operations, commercial, markdown-html, loop-library, plus standards). 663 Python tools, 746 reference guides, 103 agents (cs-* + personas), 118 slash commands across 89 marketplace plugins. v2.11.2 vendors engineering/skillopt-sleep — a verbatim copy of microsoft/SkillOpt's stdlib-only skillopt_sleep engine + Claude Code plugin surface, giving a local agent a nightly gated self-improvement cycle (read-only session harvest -> mine -> offline replay -> held-out-gated CLAUDE.md/SKILL.md edits -> staged for explicit /skillopt-sleep adopt). productivity/fable-goal (unreleased, post-v2.11.1) converts a rambling description of a desired outcome into one polished /goal prompt for a fresh autonomous session. v2.11.1 turns product-team and project-management into agent-harness domains: fork-orchestrators with deterministic goal routers, a Jira MCP snapshot bridge (Kanban flow metrics + Monte Carlo forecasting), a delegation-governance loop gate, a continuous-discovery cadence tracker, and an Opportunity Solution Tree linter, with /cs:pm and /cs:product command families. v2.10.3 completes the markdown-html domain with md-slides — slide-deck converter (arrow-key / Space / PgDn / Home/End / P keyboard navigation + presenter mode with split-view clock + speaker notes + next-slide preview + URL-hash deep linking like #3 for direct slide jumps + @media print page-per-slide for browser-native PDF export). Reuses md-document's markdown parser; vanilla JS only (no framework runtime); Prism.js opt-in via --syntax. Joins md-review (v2.10.2 code-review converter), md-document (v2.10.1 long-form converter), and the v2.10.0 foundation (orchestrator + design-system). Compatible with Claude Code, Codex CLI, Gemini CLI, Cursor, OpenClaw, Hermes Agent, Mistral Vibe, and 5 more coding agents.", "version": "2.11.2" }, "plugins": [ diff --git a/CHANGELOG.md b/CHANGELOG.md index f56d0779..275591a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,7 +22,7 @@ parsers) is vendored close to verbatim and keeps upstream's format chains, chapt detection across Latin/Roman/Chinese/Thai/Korean heading styles, invisible-Unicode (Trojan Source) sanitization, and the DOCX entity-expansion guard. -**16 numbered deviations** are recorded in `engineering/book-to-skill/README.md`, +**18 numbered deviations** are recorded in `engineering/book-to-skill/README.md`, which is the authoritative list. Highlights: - **(5) No implicit installs.** `--install-missing` defaults to `report` — it prints @@ -48,6 +48,16 @@ which is the authoritative list. Highlights: `--workdir` is symlink-refused and mode-restricted. Also fixes a real bug: `parsers/calibre.py` read a module-level path constant and so ignored `--workdir` entirely. +- **(17) Zip-of-XML hardening generalized, plus decompression-bomb caps.** Upstream's + DTD/entity guard covered DOCX only; EPUB's `ebooklib` path handed the archive straight to a + third-party XML stack. The guard now lives in `book_to_skill/zip_safety.py` and runs for + both, and every archive read checks declared size and compression ratio before + decompressing — a 200 MB zip bomb is refused at ~14 MB peak RSS. +- **(18) Packaging refuses a source tree containing symlinks.** `shutil.copytree` defaults to + following links, which would bake a link target's real content into a package that may be + emitted as `--distribution shareable`. `_assert_no_symlinks()` walks the whole tree and + refuses, before the validation branch so `--skip-validation` cannot bypass it. + **Repo-native addition with no upstream counterpart — Step 11 / `/cs:book-to-plugin`.** Upstream stops at a bare folder in `~/.claude/skills/`, which this library cannot route to. `skill_plugin_emitter.py` wraps a compiled skill as a full plugin package (manifest @@ -73,7 +83,7 @@ Regenerated `engineering/agent-harness/.../assets/harnesses/engineering.json`. P ### Changed — counters -skills 362 → 363, tools 644 → 662, refs 741 → 746, agents 102 → 103, commands +skills 362 → 363, tools 644 → 663, refs 741 → 746, agents 102 → 103, commands 116 → 118, plugins 88 → 89 (derived via `scripts/derive_counters.py --check`). --- diff --git a/CLAUDE.md b/CLAUDE.md index a8a22802..8fb27066 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co This is a **comprehensive skills library** for Claude AI and Claude Code - reusable, production-ready skill packages that bundle domain expertise, best practices, analysis tools, and strategic frameworks. The repository provides modular skills that teams can download and use directly in their workflows. -**Current Scope:** 363 production-ready skills across 18 domains with 662 Python automation tools, 746 reference guides, 103 agents (cs-* + 7 personas), and 118 slash commands, distributed as 89 marketplace plugins. Headline counters are derived from the tree by `scripts/derive_counters.py` (run with `--check` to verify the docs still match). **v2.11.2 (current)** vendors **engineering/skillopt-sleep/** — started as a verbatim, byte-for-byte copy of `microsoft/SkillOpt`'s `skillopt_sleep` engine (stdlib-only, zero third-party deps) and its Claude Code plugin surface (`skills/`, `hooks/`, `commands/`, `scripts/`), then received 23 targeted patches after ten rounds of adversarial review (see `engineering/skillopt-sleep/README.md`'s numbered "Deviations from upstream" list, the authoritative source — re-apply all 23 on re-vendor). Gives a local agent a nightly "sleep cycle": read-only harvest of past Claude Code session transcripts → mine recurring tasks → replay offline on the user's own API budget → consolidate into `CLAUDE.md`/`SKILL.md` edits behind a held-out validation gate → stage for review; nothing live changes until an explicit `/skillopt-sleep adopt` (which backs up first). Default `mock` backend spends no API budget. The heavier `skillopt` *training* package (benchmark-driven, needs `numpy`/`openai`/`azure-*` + hand-labeled train/val/test data per task) was deliberately **not** vendored — it optimizes one narrow, scoreable task at a time, which doesn't fit this repo's broad domain-expertise skills or its no-ML-in-scripts/no-test-framework conventions; `skillopt_sleep` mines its "benchmark" from real usage instead, which does fit. Attribution preserved in `plugin.json` + `LICENSE` + `README.md` (MIT, © Microsoft Corporation / Yifan Yang), following the same verbatim-vendor pattern as `loop-library/`. **Unreleased (post-v2.11.2)** ships the **productivity coverage expansion** — public audit record `audit/productivity-2026-07/` (all 7 legacy skills scored, 24/24 scripts smoke-tested, coverage map vs the personal-productivity canon) + 3 gap-filling plugins, each with a cs-* agent, /cs:* commands, 3 stdlib scripts and 3 cited references: **weekly-review** (GTD loop; review-gate refuses COMPLETE while a mandatory GET CURRENT step is missing), **deep-work** (time-block planner refusing >4h deep demand, shallow-work budget auditor, focus-session logger), **meetings** (MEET/ASYNC/NOT-READY cost gate, outcome-required agenda builder, action-item extractor with ORPHAN/NO-DUE flags). **Unreleased (post-v2.11.1)** added **productivity/fable-goal** — converts a rambling description of a desired outcome into one polished, copy-paste `/goal` prompt for a fresh autonomous session (ported from `duncan-buildroom/freeskills`). **v2.11.1 (complete)** upgrades **product-team/** and **project-management/** into agent-harness domains: both prose routers rebuilt as `context: fork` orchestrators with deterministic goal routers (exit-code route/ask/refuse), a Jira MCP snapshot bridge (Kanban-Guide-2025 flow metrics + seeded Monte Carlo forecasts, verified end-to-end into velocity_analyzer), a delegation-governance loop gate (human owner / reviewer / machine-checkable acceptance / close refusal), a Torres continuous-discovery cadence tracker + Opportunity Solution Tree linter, cs-pm-orchestrator + cs-product-orchestrator agents, and /cs:pm|grill-pm|pm-loop + /cs:product|grill-product|product-loop commands — plus the public audit record `audit/pm-product-agentic-2026-07/` (AR-rubric scores for all 26 skills, research-backed improvement fields, executable verification criteria). **v2.9.0 (complete)** added the **research-ops/** top-level domain — enterprise Research Operations (orchestrator + clinical-research + research-finance + market-research + product-research), the managed counterpart to the academic research/ domain, with `context: fork` orchestration and a Matt Pocock "Forcing-question library" in every SKILL.md plus `/cs:grill-research-ops`. **v2.8.0 (complete)** added 2 new top-level domains — **business-operations/** (7 internal-ops skills: orchestrator + process-mapper + vendor-management + capacity-planner + internal-comms + knowledge-ops + procurement-optimizer) and **commercial/** (8 per-deal-economics skills: orchestrator + pricing-strategist + deal-desk + partnerships-architect + channel-economics + commercial-policy + rfp-responder + commercial-forecaster) — with orchestrator skills using `context: fork` for chaining, Matt Pocock docs-anchored "Forcing-question library" in every SKILL.md, plus `/cs:grill-bizops` and `/cs:grill-commercial`. **v2.8.2** adds a productivity-shaped `handoff` skill (sibling to engineering/handoff) inspired by Matt Pocock — first-run setup with configurable save location, redaction linter, SessionStart + SessionEnd hooks, fidelity self-check, `--refresh` flag. **v2.8.1** upgraded the engineering role-skills (senior-fullstack / senior-frontend / senior-backend) with karpathy-coder + Matt Pocock decision engines + per-role forcing questions. v2.7.3 ports `alirezarezvani/aeo-box` — AEO (Answer Engine Optimization) skill into marketing-skill/ + security-guidance PreToolUse hook into engineering/. v2.7.0 added 13 Path-B skills across 3 top-level domains (productivity, marketing, research). v2.6.0 added 4 Matt Pocock-derived productivity skills. +**Current Scope:** 363 production-ready skills across 18 domains with 663 Python automation tools, 746 reference guides, 103 agents (cs-* + 7 personas), and 118 slash commands, distributed as 89 marketplace plugins. Headline counters are derived from the tree by `scripts/derive_counters.py` (run with `--check` to verify the docs still match). **v2.11.2 (current)** vendors **engineering/skillopt-sleep/** — started as a verbatim, byte-for-byte copy of `microsoft/SkillOpt`'s `skillopt_sleep` engine (stdlib-only, zero third-party deps) and its Claude Code plugin surface (`skills/`, `hooks/`, `commands/`, `scripts/`), then received 23 targeted patches after ten rounds of adversarial review (see `engineering/skillopt-sleep/README.md`'s numbered "Deviations from upstream" list, the authoritative source — re-apply all 23 on re-vendor). Gives a local agent a nightly "sleep cycle": read-only harvest of past Claude Code session transcripts → mine recurring tasks → replay offline on the user's own API budget → consolidate into `CLAUDE.md`/`SKILL.md` edits behind a held-out validation gate → stage for review; nothing live changes until an explicit `/skillopt-sleep adopt` (which backs up first). Default `mock` backend spends no API budget. The heavier `skillopt` *training* package (benchmark-driven, needs `numpy`/`openai`/`azure-*` + hand-labeled train/val/test data per task) was deliberately **not** vendored — it optimizes one narrow, scoreable task at a time, which doesn't fit this repo's broad domain-expertise skills or its no-ML-in-scripts/no-test-framework conventions; `skillopt_sleep` mines its "benchmark" from real usage instead, which does fit. Attribution preserved in `plugin.json` + `LICENSE` + `README.md` (MIT, © Microsoft Corporation / Yifan Yang), following the same verbatim-vendor pattern as `loop-library/`. **Unreleased (post-v2.11.2)** ships the **productivity coverage expansion** — public audit record `audit/productivity-2026-07/` (all 7 legacy skills scored, 24/24 scripts smoke-tested, coverage map vs the personal-productivity canon) + 3 gap-filling plugins, each with a cs-* agent, /cs:* commands, 3 stdlib scripts and 3 cited references: **weekly-review** (GTD loop; review-gate refuses COMPLETE while a mandatory GET CURRENT step is missing), **deep-work** (time-block planner refusing >4h deep demand, shallow-work budget auditor, focus-session logger), **meetings** (MEET/ASYNC/NOT-READY cost gate, outcome-required agenda builder, action-item extractor with ORPHAN/NO-DUE flags). **Unreleased (post-v2.11.1)** added **productivity/fable-goal** — converts a rambling description of a desired outcome into one polished, copy-paste `/goal` prompt for a fresh autonomous session (ported from `duncan-buildroom/freeskills`). **v2.11.1 (complete)** upgrades **product-team/** and **project-management/** into agent-harness domains: both prose routers rebuilt as `context: fork` orchestrators with deterministic goal routers (exit-code route/ask/refuse), a Jira MCP snapshot bridge (Kanban-Guide-2025 flow metrics + seeded Monte Carlo forecasts, verified end-to-end into velocity_analyzer), a delegation-governance loop gate (human owner / reviewer / machine-checkable acceptance / close refusal), a Torres continuous-discovery cadence tracker + Opportunity Solution Tree linter, cs-pm-orchestrator + cs-product-orchestrator agents, and /cs:pm|grill-pm|pm-loop + /cs:product|grill-product|product-loop commands — plus the public audit record `audit/pm-product-agentic-2026-07/` (AR-rubric scores for all 26 skills, research-backed improvement fields, executable verification criteria). **v2.9.0 (complete)** added the **research-ops/** top-level domain — enterprise Research Operations (orchestrator + clinical-research + research-finance + market-research + product-research), the managed counterpart to the academic research/ domain, with `context: fork` orchestration and a Matt Pocock "Forcing-question library" in every SKILL.md plus `/cs:grill-research-ops`. **v2.8.0 (complete)** added 2 new top-level domains — **business-operations/** (7 internal-ops skills: orchestrator + process-mapper + vendor-management + capacity-planner + internal-comms + knowledge-ops + procurement-optimizer) and **commercial/** (8 per-deal-economics skills: orchestrator + pricing-strategist + deal-desk + partnerships-architect + channel-economics + commercial-policy + rfp-responder + commercial-forecaster) — with orchestrator skills using `context: fork` for chaining, Matt Pocock docs-anchored "Forcing-question library" in every SKILL.md, plus `/cs:grill-bizops` and `/cs:grill-commercial`. **v2.8.2** adds a productivity-shaped `handoff` skill (sibling to engineering/handoff) inspired by Matt Pocock — first-run setup with configurable save location, redaction linter, SessionStart + SessionEnd hooks, fidelity self-check, `--refresh` flag. **v2.8.1** upgraded the engineering role-skills (senior-fullstack / senior-frontend / senior-backend) with karpathy-coder + Matt Pocock decision engines + per-role forcing questions. v2.7.3 ports `alirezarezvani/aeo-box` — AEO (Answer Engine Optimization) skill into marketing-skill/ + security-guidance PreToolUse hook into engineering/. v2.7.0 added 13 Path-B skills across 3 top-level domains (productivity, marketing, research). v2.6.0 added 4 Matt Pocock-derived productivity skills. **Key Distinction**: This is NOT a traditional application. It's a library of skill packages meant to be extracted and deployed by users into their own Claude workflows. @@ -177,10 +177,10 @@ Vendors `engineering/skillopt-sleep/` — a byte-for-byte start from [microsoft/ Derived from [virgiliojr94/book-to-skill](https://github.com/virgiliojr94/book-to-skill) (MIT). Compiles a book, documentation folder, or spec collection (PDF, EPUB, DOCX, HTML, Markdown, RST, AsciiDoc, RTF, MOBI/AZW) into an agent skill: a resident master `SKILL.md` (core frameworks + chapter index + topic index, capped at 4k tokens) plus on-demand `chapters/chNN-*.md`, `glossary.md`, `patterns.md`, and a decision `cheatsheet.md`. The agent reads the core, then one chapter — never the whole source again. - **Vendored close to verbatim:** the extraction library (`scripts/book_to_skill/` — config, exceptions, sanitize, dependencies, utils + 7 per-format parsers) keeps upstream's format chains, chapter detection across Latin/Roman/Chinese/Thai/Korean heading styles, invisible-Unicode (Trojan Source) sanitization and DOCX entity-expansion guard. -- **16 numbered deviations from upstream** — the list in `engineering/book-to-skill/README.md` is authoritative; `plugin.json`'s `attribution.derivation_note` summarizes it. Highlights: (5) `--install-missing` now defaults to `report` — it prints the pip command and uses the stdlib fallback instead of upstream's TTY prompt that runs `pip install` into the caller's environment; (6) a **rights gate** — `skill_plugin_emitter.py --distribution shareable` refuses without `--rights` from `public-domain|open-license|internal-docs|author-permission`, with `fair-use` deliberately excluded (a defence, not a licence); (10) the two upstream validators merged into one four-family gate, adding **budget** and **index** families — dead chapter links, unindexed chapter files and dangling topic refs are the failure that silently breaks navigation while the skill still looks complete, and upstream had no check for it; (11) folded YAML scalars now parse, so a wrapped description no longer under-reports its length past the 1024-char cap; (12) `discovery_tax.py` → `token_budget_estimator.py` with the optional `tiktoken` path dropped, a post-flight budget audit added, and an explicit **worth-converting verdict** that says "just read it" when the source is under ~3× the compiled skill. +- **18 numbered deviations from upstream** — the list in `engineering/book-to-skill/README.md` is authoritative; `plugin.json`'s `attribution.derivation_note` summarizes it. Highlights: (5) `--install-missing` now defaults to `report` — it prints the pip command and uses the stdlib fallback instead of upstream's TTY prompt that runs `pip install` into the caller's environment; (6) a **rights gate** — `skill_plugin_emitter.py --distribution shareable` refuses without `--rights` from `public-domain|open-license|internal-docs|author-permission`, with `fair-use` deliberately excluded (a defence, not a licence); (10) the two upstream validators merged into one four-family gate, adding **budget** and **index** families — dead chapter links, unindexed chapter files and dangling topic refs are the failure that silently breaks navigation while the skill still looks complete, and upstream had no check for it; (11) folded YAML scalars now parse, so a wrapped description no longer under-reports its length past the 1024-char cap; (12) `discovery_tax.py` → `token_budget_estimator.py` with the optional `tiktoken` path dropped, a post-flight budget audit added, and an explicit **worth-converting verdict** that says "just read it" when the source is under ~3× the compiled skill. - **Repo-native addition with no upstream counterpart — Step 11 / `/cs:book-to-plugin`:** upstream stops at a bare folder in `~/.claude/skills/`, which this library cannot route to. `skill_plugin_emitter.py` wraps a compiled skill as a full plugin package (manifest + `cs-` agent + `/cs:` command + README) and prints the marketplace entry; it never edits `marketplace.json` itself, and refuses to wrap a skill carrying validation errors. - **Cross-linked into `engineering/write-a-skill`** ("author first, compile second" — that skill authors from expertise in your head, this one compiles from a document on disk). -- 4 stdlib-only tools (all `--help` / `--sample` / `--output json`), 5 references citing 7–8 sources each, 3 assets, `cs-book-to-skill` agent, 2 commands. **Counters:** skills 362 → 363; tools 644 → 662; refs 741 → 746; agents 102 → 103; commands 116 → 118; plugins 88 → 89 (derived via `scripts/derive_counters.py --check`). +- 4 stdlib-only tools (all `--help` / `--sample` / `--output json`), 5 references citing 7–8 sources each, 3 assets, `cs-book-to-skill` agent, 2 commands. **Counters:** skills 362 → 363; tools 644 → 663; refs 741 → 746; agents 102 → 103; commands 116 → 118; plugins 88 → 89 (derived via `scripts/derive_counters.py --check`). --- diff --git a/README.md b/README.md index 9b0e21b1..4137d6fa 100644 --- a/README.md +++ b/README.md @@ -26,10 +26,10 @@ The most comprehensive open-source library of Claude Code skills and agent plugi Claude Code skills (also called agent skills or coding agent plugins) are modular instruction packages that give AI coding agents domain expertise they don't have out of the box. Each skill includes: - **SKILL.md** — structured instructions, workflows, and decision frameworks -- **Python tools** — 662 CLI scripts (all stdlib-only, zero pip installs) +- **Python tools** — 663 CLI scripts (all stdlib-only, zero pip installs) - **Reference docs** — 746 templates, checklists, and domain-specific knowledge files -**One repo, thirteen platforms.** Works natively as Claude Code plugins, Codex agent skills, Gemini CLI skills, Hermes Agent skills, Mistral Vibe skills, and converts to more tools via `scripts/convert.sh`. All 662 Python tools run anywhere Python runs. +**One repo, thirteen platforms.** Works natively as Claude Code plugins, Codex agent skills, Gemini CLI skills, Hermes Agent skills, Mistral Vibe skills, and converts to more tools via `scripts/convert.sh`. All 663 Python tools run anywhere Python runs. ### Skills vs Agents vs Personas @@ -354,7 +354,7 @@ Yes. Skills work natively with 13 tools: Claude Code, OpenAI Codex, Gemini CLI, No. We follow semantic versioning and maintain backward compatibility within patch releases. Existing script arguments, plugin source paths, and SKILL.md structures are never changed in patch versions. See the [CHANGELOG](CHANGELOG.md) for details on each release. **Are the Python tools dependency-free?** -Yes. All 662 Python tools use the standard library only — zero pip installs required. Every skill's CLI entry point is verified to run with `--help` (most skills ship one script per tool; a few, like the vendored `engineering/skillopt-sleep` engine, ship a multi-module package behind a single `python -m` entry point). A few tools — `engineering/book-to-skill`'s document extractors — can *optionally* use third-party parsers for higher-fidelity output, but every format falls back to a standard-library parser and nothing is installed implicitly. +Yes. All 663 Python tools use the standard library only — zero pip installs required. Every skill's CLI entry point is verified to run with `--help` (most skills ship one script per tool; a few, like the vendored `engineering/skillopt-sleep` engine, ship a multi-module package behind a single `python -m` entry point). A few tools — `engineering/book-to-skill`'s document extractors — can *optionally* use third-party parsers for higher-fidelity output, but every format falls back to a standard-library parser and nothing is installed implicitly. **How do I create my own Claude Code skill?** Each skill is a folder with a `SKILL.md` (frontmatter + instructions), optional `scripts/`, `references/`, and `assets/`. See the [Skills & Agents Factory](https://github.com/alirezarezvani/claude-code-skills-agents-factory) for a step-by-step guide. diff --git a/engineering/book-to-skill/.claude-plugin/plugin.json b/engineering/book-to-skill/.claude-plugin/plugin.json index fe639303..4be9b76f 100644 --- a/engineering/book-to-skill/.claude-plugin/plugin.json +++ b/engineering/book-to-skill/.claude-plugin/plugin.json @@ -16,6 +16,6 @@ "derived_from": "https://github.com/virgiliojr94/book-to-skill", "original_author": "virgiliojr94", "original_license": "MIT", - "derivation_note": "The extraction library (book_to_skill/ — config, exceptions, sanitize, dependencies, utils, and the 7 per-format parsers) is vendored from upstream and carries upstream's multi-format chains, chapter detection across 5 script families, and Unicode/XXE hardening. 16 numbered deviations are recorded in README.md, which is the authoritative list: the workflow was rewritten Claude-Code-first for this repo's conventions, the 3 upstream tools were reworked into 4 argparse CLIs with --help/--sample, install-on-import behaviour was replaced with a report-only default, and Step 11 (plugin emission with a rights gate) plus the validator's budget and index families are additions with no upstream counterpart." + "derivation_note": "The extraction library (book_to_skill/ — config, exceptions, sanitize, dependencies, utils, and the 7 per-format parsers) is vendored from upstream and carries upstream's multi-format chains, chapter detection across 5 script families, and Unicode/XXE hardening. 18 numbered deviations are recorded in README.md, which is the authoritative list: the workflow was rewritten Claude-Code-first for this repo's conventions, the 3 upstream tools were reworked into 4 argparse CLIs with --help/--sample, install-on-import behaviour was replaced with a report-only default, and Step 11 (plugin emission with a rights gate) plus the validator's budget and index families are additions with no upstream counterpart." } } diff --git a/engineering/book-to-skill/README.md b/engineering/book-to-skill/README.md index 1e7fa6b0..398d771d 100644 --- a/engineering/book-to-skill/README.md +++ b/engineering/book-to-skill/README.md @@ -210,6 +210,26 @@ small; just read it." (`SKILL_FILE_BUDGETS`, `CHAPTER_TOKEN_CEILING`) rather than being restated in each tool, where they would drift the first time a cap changed. +17. **Zip-of-XML hardening generalized to EPUB, plus decompression-bomb caps.** Upstream + hardened DOCX and only DOCX: `validate_docx_xml_safety()` screened that archive for + DTD/entity declarations before any parser touched it. EPUB is the same shape — a zip whose + members are XML — and its `ebooklib` path handed the file straight to a third-party XML + stack with no equivalent check, despite `ebooklib` being one of the packages this skill + recommends installing. The guard now lives in `book_to_skill/zip_safety.py` and runs for + both formats. Every archive read also goes through `safe_read()`, which consults the + declared uncompressed size and the compression ratio *before* decompressing: a 200 MB + zip bomb is refused at ~14 MB peak RSS instead of being materialized. Neither parser ever + writes archive members to disk, so zip-slip stays out of scope by construction. + +18. **Packaging refuses a source tree containing symlinks.** The validator checks the files it + knows about (`SKILL.md`, the three supporting files, `chapters/*.md`), but `shutil.copytree` + defaults to `symlinks=False` and follows a link *anywhere else* in the tree — an `assets/` + entry, any subdirectory — baking the target's real content into a package that may then be + emitted as `--distribution shareable`. `_assert_no_symlinks()` now walks the whole tree and + refuses, and it runs **before** the validation branch so `--skip-validation` cannot bypass + it. `copytree` also passes `symlinks=True` so a future edit loosening that check cannot + silently reintroduce dereferencing. + --- ## Security audit diff --git a/engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/docx.py b/engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/docx.py index 879d8d14..c82ec746 100644 --- a/engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/docx.py +++ b/engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/docx.py @@ -3,6 +3,7 @@ from __future__ import annotations import zipfile import sys from book_to_skill.exceptions import ExtractionError +from book_to_skill.zip_safety import safe_read, validate_zip_xml_safety def extract_docx_with_python_docx(docx_path: str) -> str | None: @@ -28,7 +29,7 @@ def extract_docx_with_zipfile(docx_path: str) -> str | None: import xml.etree.ElementTree as ET with zipfile.ZipFile(docx_path) as zf: - xml_bytes = zf.read("word/document.xml") + xml_bytes = safe_read(zf, "word/document.xml") root = ET.fromstring(xml_bytes) ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}" parts: list[str] = [] @@ -69,27 +70,13 @@ def extract_docx_with_zipfile(docx_path: str) -> str | None: def validate_docx_xml_safety(docx_path: str) -> None: - """Scan all XML files in the DOCX zip archive to prevent XML Entity Expansion (Billion Laughs) and XXE injections.""" - try: - with zipfile.ZipFile(docx_path) as zf: - for name in zf.namelist(): - if name.endswith(".xml") or name.endswith(".rels"): - xml_bytes = zf.read(name) - for encoding in ("utf-8", "utf-16", "utf-16le", "utf-16be", "utf-32"): - try: - content = xml_bytes.decode(encoding, errors="ignore").upper() - except LookupError: - continue - if " tuple[str, str]: diff --git a/engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/epub.py b/engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/epub.py index ffd25c2a..80ec53db 100644 --- a/engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/epub.py +++ b/engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/epub.py @@ -5,9 +5,19 @@ import re import zipfile import sys from book_to_skill.parsers.html_text import _HTMLTextExtractor +from book_to_skill.zip_safety import _Budget, safe_read, validate_zip_xml_safety def extract_with_ebooklib(epub_path: str) -> str | None: + """Extract via ebooklib, after the same XML guard DOCX gets. + + ebooklib parses container.xml, the OPF package document and the content docs + with a third-party XML stack whose entity handling this code does not + control. The archive is an untrusted file the user was handed, so it is + screened for DTD/entity declarations before ebooklib sees it — upstream ran + this check on DOCX only. + """ + validate_zip_xml_safety(epub_path, label="EPUB") try: import ebooklib from ebooklib import epub @@ -34,7 +44,7 @@ def _find_opf_path(zf: zipfile.ZipFile) -> str | None: """ # Spec-defined: read container.xml for the rootfile path try: - container = zf.read("META-INF/container.xml").decode("utf-8", errors="replace") + container = safe_read(zf, "META-INF/container.xml").decode("utf-8", errors="replace") match = re.search(r'full-path=["\']([^"\']+\.opf)["\']', container) if match: return match.group(1) @@ -50,6 +60,7 @@ def extract_with_zipfile(epub_path: str) -> str | None: """stdlib-only EPUB extractor: unzip → parse HTML files.""" try: with zipfile.ZipFile(epub_path) as zf: + budget = _Budget() names = zf.namelist() # Locate OPF and determine its directory for resolving relative hrefs @@ -61,7 +72,7 @@ def extract_with_zipfile(epub_path: str) -> str | None: spine_order: list[str] = [] seen: set[str] = set() if opf_path: - opf_text = zf.read(opf_path).decode("utf-8", errors="replace") + opf_text = safe_read(zf, opf_path, budget).decode("utf-8", errors="replace") # Manifest: item id -> resolved href. Parse each opening # tag so attribute order (id before/after href) does not matter; @@ -99,7 +110,7 @@ def extract_with_zipfile(epub_path: str) -> str | None: parts = [] for name in html_files: try: - raw = zf.read(name).decode("utf-8", errors="replace") + raw = safe_read(zf, name, budget).decode("utf-8", errors="replace") parser = _HTMLTextExtractor() parser.feed(raw) parts.append(parser.get_text()) @@ -118,7 +129,7 @@ def count_epub_chapters(epub_path: str) -> int: opf_path = _find_opf_path(zf) if not opf_path: return 0 - opf_text = zf.read(opf_path).decode("utf-8", errors="replace") + opf_text = safe_read(zf, opf_path).decode("utf-8", errors="replace") return len(re.findall(r'` declarations drive billion-laughs + memory exhaustion; external entities read local files into the parsed output. + Python's `xml.etree.ElementTree` does not resolve *external* entities, but it + does expand internal ones — and third-party parsers built on lxml vary. The + check refuses the archive rather than trusting each parser's defaults. + +2. **Decompression bombs.** A few KB of zip can expand to gigabytes. Every read + goes through `safe_read()`, which consults the declared uncompressed size and + the compression ratio *before* decompressing, so a bomb is refused rather + than materialized. + +Neither parser ever writes archive members to disk (no `extractall`/`extract`), +so zip-slip is out of scope by construction. +""" + +from __future__ import annotations + +import zipfile + +from book_to_skill.exceptions import ExtractionError + +# A single member of a book archive. Generous for a chapter's XHTML or a +# document.xml, far below what a bomb needs to hurt. +MAX_MEMBER_BYTES = 64 * 1024 * 1024 +# Everything read from one archive, across all members. +MAX_TOTAL_BYTES = 512 * 1024 * 1024 +# Declared-uncompressed / stored ratio. Real prose and XML land well under 100x; +# the classic zip bomb is ~1000x and up. +MAX_COMPRESSION_RATIO = 200 + +# Members worth scanning for DTD/entity declarations: XML by extension, plus the +# extensionless `mimetype` member and OPF/NCX which are XML without an .xml suffix. +_XML_SUFFIXES = (".xml", ".rels", ".opf", ".ncx", ".xhtml", ".html", ".htm") + + +class _Budget: + """Running total of bytes decompressed from one archive.""" + + def __init__(self, limit: int = MAX_TOTAL_BYTES) -> None: + self.limit = limit + self.used = 0 + + def charge(self, size: int, name: str) -> None: + self.used += size + if self.used > self.limit: + raise ExtractionError( + f"archive exceeds the {self.limit:,}-byte decompression budget " + f"(reading '{name}') — refusing to continue" + ) + + +def safe_read(zf: zipfile.ZipFile, name: str, budget: _Budget | None = None) -> bytes: + """Read one archive member after checking its declared size and ratio. + + The checks run against the zip's central directory *before* decompressing, + so a bomb never gets materialized. A liar in the directory still cannot get + past the budget, because the actual read is charged against it too. + """ + try: + info = zf.getinfo(name) + except KeyError as exc: + raise ExtractionError(f"archive member not found: {name}") from exc + + if info.file_size > MAX_MEMBER_BYTES: + raise ExtractionError( + f"archive member '{name}' declares {info.file_size:,} bytes uncompressed; " + f"the per-member limit is {MAX_MEMBER_BYTES:,} bytes" + ) + if info.compress_size > 0: + ratio = info.file_size / info.compress_size + if ratio > MAX_COMPRESSION_RATIO: + raise ExtractionError( + f"archive member '{name}' expands {ratio:.0f}x " + f"({info.compress_size:,} -> {info.file_size:,} bytes); the limit is " + f"{MAX_COMPRESSION_RATIO}x — this looks like a decompression bomb" + ) + + data = zf.read(name) + if budget is not None: + budget.charge(len(data), name) + return data + + +def is_xml_member(name: str) -> bool: + return name.lower().endswith(_XML_SUFFIXES) + + +def validate_zip_xml_safety(archive_path: str, label: str = "archive") -> None: + """Refuse a zip-of-XML archive that declares a DTD or any entity. + + Scans every XML-ish member across the encodings a hostile file might use to + hide the declaration from a naive UTF-8 substring search. Generalized from + upstream's DOCX-only guard so EPUB gets the same treatment. + """ + try: + with zipfile.ZipFile(archive_path) as zf: + budget = _Budget() + for name in zf.namelist(): + if not is_xml_member(name): + continue + xml_bytes = safe_read(zf, name, budget) + for encoding in ("utf-8", "utf-16", "utf-16le", "utf-16be", "utf-32"): + try: + content = xml_bytes.decode(encoding, errors="ignore").upper() + except LookupError: + continue + if " ) +def _assert_no_symlinks(skill_dir: Path) -> None: + """Refuse to package a source tree containing any symbolic link. + + The validator checks the files it knows about — SKILL.md, the three + supporting files, chapters/*.md — but `shutil.copytree` defaults to + `symlinks=False`, which follows a link *anywhere else* in the tree (an + `assets/` entry, an arbitrary subdirectory) and bakes the target's real + content into the emitted package. That package can then go out as + `--distribution shareable`, so a link pointing at something outside the + skill becomes part of a published artifact. + + Refusing beats `copytree(symlinks=True)`: preserving the link would ship a + package whose contents depend on the emitting machine's filesystem. This + check runs even under `--skip-validation`, which otherwise disables the + file-level symlink checks entirely. + """ + offenders = [p for p in sorted(skill_dir.rglob("*")) if p.is_symlink()] + if offenders: + listed = "\n".join(f" {p.relative_to(skill_dir).as_posix()} -> {os.readlink(p)}" + for p in offenders[:10]) + more = f"\n ... and {len(offenders) - 10} more" if len(offenders) > 10 else "" + raise EmitError( + f"source skill contains {len(offenders)} symbolic link(s); packaging would copy " + f"the target's content into a distributable package:\n{listed}{more}\n" + "Replace them with real files, or point --skill-dir at a tree without links." + ) + + def _plugin_manifest(identity: dict, *, domain: str, author: str, author_url: str, repository: str, distribution: str, rights: str | None, source_note: str) -> dict: @@ -337,6 +366,10 @@ def emit(*, skill_dir: Path, dest_root: Path, domain: str, author: str, author_u "Or emit with --distribution local (the default) to keep it on this machine." ) + # Unconditional: --skip-validation waives content findings, never the + # guarantee that packaging copies only what is actually in the source tree. + _assert_no_symlinks(skill_dir) + if not skip_validation: try: findings = validate(skill_dir) @@ -389,7 +422,9 @@ def emit(*, skill_dir: Path, dest_root: Path, domain: str, author: str, author_u target = package_root / relative target.parent.mkdir(parents=True, exist_ok=True) target.write_text(text, encoding="utf-8") - shutil.copytree(skill_dir, package_root / "skills" / identity["slug"]) + # symlinks=True is redundant after _assert_no_symlinks, and kept so a future + # edit that loosens that check cannot silently reintroduce dereferencing. + shutil.copytree(skill_dir, package_root / "skills" / identity["slug"], symlinks=True) return result diff --git a/scripts/smoke_exceptions.txt b/scripts/smoke_exceptions.txt index 12c07ce8..3a119965 100644 --- a/scripts/smoke_exceptions.txt +++ b/scripts/smoke_exceptions.txt @@ -83,6 +83,7 @@ engineering/skillopt-sleep/skillopt_sleep/types.py # internal module; module it engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/__init__.py # package init, not a CLI engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/dependencies.py # internal module engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/utils.py # internal module +engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/zip_safety.py # internal module engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/calibre.py # internal module engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/docx.py # internal module engineering/book-to-skill/skills/book-to-skill/scripts/book_to_skill/parsers/epub.py # internal module