scriptoriumadmin/claude-skills
1
0
Fork 0
mirror of https://github.com/alirezarezvani/claude-skills.git synced 2026-08-28 04:24:58 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #996 from alirezarezvani/dev
Some checks failed
Deploy Documentation to Pages / build (push) Has been cancelled
Details
Sync Codex Skills Symlinks / sync (push) Has been cancelled
Details
Release / Tag + GitHub Release (push) Has been cancelled
Details
Deploy Documentation to Pages / deploy (push) Has been cancelled
Details

Browse source
This commit is contained in:
Alireza Rezvani 2026-08-26 15:53:02 +02:00 • committed by GitHub
commit 19392f7a08
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
92 changed files with 7710 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
.claude-plugin/marketplace.json
View file

@ -8,7 +8,7 @@
"homepage": "https://github.com/alirezarezvani/claude-skills",
"repository": "https://github.com/alirezarezvani/claude-skills",
"metadata": {
"description": "386 production-ready skills across 20 domains (engineering, engineering-core, marketing, product, c-level, c-level-agents, compliance-os, project management, RA/QM, business growth, finance, productivity, marketing top-level, research, research-ops, business-operations, commercial, markdown-html, loop-library, plus standards). 723 Python tools, 838 reference guides, 116 agents (cs-* + personas), 146 slash commands across 97 marketplace plugins. v2.11.2 vendors engineering/skillopt-sleep — a verbatim copy of microsoft/SkillOpt's stdlib-only skillopt_sleep engine + Claude Code plugin surface, giving a local agent a nightly gated self-improvement cycle (read-only session harvest -> mine -> offline replay -> held-out-gated CLAUDE.md/SKILL.md edits -> staged for explicit /skillopt-sleep adopt). productivity/fable-goal (unreleased, post-v2.11.1) converts a rambling description of a desired outcome into one polished /goal prompt for a fresh autonomous session. v2.11.1 turns product-team and project-management into agent-harness domains: fork-orchestrators with deterministic goal routers, a Jira MCP snapshot bridge (Kanban flow metrics + Monte Carlo forecasting), a delegation-governance loop gate, a continuous-discovery cadence tracker, and an Opportunity Solution Tree linter, with /cs:pm and /cs:product command families. v2.10.3 completes the markdown-html domain with md-slides — slide-deck converter (arrow-key / Space / PgDn / Home/End / P keyboard navigation + presenter mode with split-view clock + speaker notes + next-slide preview + URL-hash deep linking like #3 for direct slide jumps + @media print page-per-slide for browser-native PDF export). Reuses md-document's markdown parser; vanilla JS only (no framework runtime); Prism.js opt-in via --syntax. Joins md-review (v2.10.2 code-review converter), md-document (v2.10.1 long-form converter), and the v2.10.0 foundation (orchestrator + design-system). Compatible with Claude Code, Codex CLI, Gemini CLI, Cursor, OpenClaw, Hermes Agent, Mistral Vibe, and 5 more coding agents.",
"description": "388 production-ready skills across 20 domains (engineering, engineering-core, marketing, product, c-level, c-level-agents, compliance-os, project management, RA/QM, business growth, finance, productivity, marketing top-level, research, research-ops, business-operations, commercial, markdown-html, loop-library, plus standards). 727 Python tools, 842 reference guides, 118 agents (cs-* + personas), 150 slash commands across 99 marketplace plugins. v2.11.2 vendors engineering/skillopt-sleep — a verbatim copy of microsoft/SkillOpt's stdlib-only skillopt_sleep engine + Claude Code plugin surface, giving a local agent a nightly gated self-improvement cycle (read-only session harvest -> mine -> offline replay -> held-out-gated CLAUDE.md/SKILL.md edits -> staged for explicit /skillopt-sleep adopt). productivity/fable-goal (unreleased, post-v2.11.1) converts a rambling description of a desired outcome into one polished /goal prompt for a fresh autonomous session. v2.11.1 turns product-team and project-management into agent-harness domains: fork-orchestrators with deterministic goal routers, a Jira MCP snapshot bridge (Kanban flow metrics + Monte Carlo forecasting), a delegation-governance loop gate, a continuous-discovery cadence tracker, and an Opportunity Solution Tree linter, with /cs:pm and /cs:product command families. v2.10.3 completes the markdown-html domain with md-slides — slide-deck converter (arrow-key / Space / PgDn / Home/End / P keyboard navigation + presenter mode with split-view clock + speaker notes + next-slide preview + URL-hash deep linking like #3 for direct slide jumps + @media print page-per-slide for browser-native PDF export). Reuses md-document's markdown parser; vanilla JS only (no framework runtime); Prism.js opt-in via --syntax. Joins md-review (v2.10.2 code-review converter), md-document (v2.10.1 long-form converter), and the v2.10.0 foundation (orchestrator + design-system). Compatible with Claude Code, Codex CLI, Gemini CLI, Cursor, OpenClaw, Hermes Agent, Mistral Vibe, and 5 more coding agents.",
"version": "2.12.0"
},
"plugins": [
@ -2050,6 +2050,49 @@
"engineering"
],
"category": "development"
},
{
"name": "spinning-up-deep-rl",
"source": "./engineering/spinning-up-deep-rl",
"description": "Knowledge base compiled from OpenAI's Spinning Up in Deep RL (MIT, Joshua Achiam) by engineering/book-to-skill. A resident core carries the RL optimization problem, the model-free taxonomy, the policy-gradient template with its five valid weights, the safe-step family (VPG to TRPO to PPO) and the overestimation family (DDPG to TD3 and SAC); 20 on-demand chapters cover key concepts and MDPs, the algorithm taxonomy and model bias, the policy gradient derivation with the log-derivative trick and EGLP lemma, Achiam's researcher essay, the key-papers topic map, the exercises including the silent DDPG broadcasting bug, the benchmark parity disclosure, one chapter per algorithm, and the logger/MPI/ExperimentGrid utilities. Ships a glossary, a patterns file with 16 techniques, a decision cheatsheet with thresholds, a cs-spinning-up-deep-rl agent and /cs:spinning-up-deep-rl. Structured study notes, not a reproduction of the source.",
"version": "1.0.0",
"author": {
"name": "Alireza Rezvani"
},
"keywords": [
"knowledge-base",
"book-to-skill",
"reinforcement-learning",
"deep-rl",
"policy-gradient",
"ppo",
"sac",
"td3",
"openai-spinning-up",
"engineering"
],
"category": "development"
},
{
"name": "deep-learning-book",
"source": "./engineering/deep-learning-book",
"description": "Study companion for the Deep Learning textbook by Goodfellow, Bengio & Courville (MIT Press, 2016), free to read at deeplearningbook.org. Twenty chapter files, a glossary, patterns and a cheatsheet index the whole book, and a delta reference dates it against 2026 practice with per-claim confidence levels: double descent qualifying the U-curve, AdamW splitting weight decay from L2, transformers displacing Chapter 10's recurrence, diffusion growing out of Chapter 18's score matching. Four stdlib tools make it executable — a prerequisite-closed reading-path planner that refuses goals the book does not cover, a training diagnostic running Chapter 11's rules in priority order so a NaN is never reported as overfitting, a capacity planner that ranks 'shrink the model' last when overparameterized, and a parameter/FLOP/activation-memory calculator that refuses a stack whose shapes do not connect. A companion, not a compilation: the book is copyrighted, so nothing here reproduces its text.",
"version": "2.12.0",
"author": {
"name": "Alireza Rezvani"
},
"keywords": [
"deep-learning",
"machine-learning",
"study-companion",
"goodfellow",
"neural-networks",
"training-diagnostics",
"optimization",
"generative-models",
"engineering"
],
"category": "development"
}
]
}

8
.codex-plugin/plugin.json
View file

@ -1,7 +1,7 @@
{
"name": "claude-code-skills",
"version": "2.2.0",
"description": "223 production-ready skills, 23 agents, and 298 Python tools across 9 domains — engineering, marketing, product, compliance, C-level advisory, and more. The largest open-source skills library for AI coding agents.",
"version": "2.12.0",
"description": "388 production-ready skills across 20 domains — engineering, marketing, product, compliance, C-level advisory, research, business operations, and more. 727 Python tools, 842 reference guides, 118 agents (cs-* + personas), 150 slash commands, 99 marketplace plugins. The largest open-source skills library for AI coding agents.",
"author": {
"name": "Alireza Rezvani",
"url": "https://alirezarezvani.com"
@ -27,8 +27,8 @@
"type": "cli",
"composerIcon": "./assets/icon.png",
"displayName": "Claude Code Skills",
"shortDescription": "223 production-ready skills for AI coding agents across 9 domains",
"longDescription": "The largest open-source skills library for AI coding agents. 223 skills covering engineering (architecture, DevOps, security, AI/ML), marketing (SEO, CRO, content), product management, C-level advisory, regulatory compliance (ISO 13485, SOC 2, GDPR), project management, business growth, and finance. Includes 298 stdlib-only Python CLI tools, 416 reference guides, 23 orchestration agents, and 22 slash commands. Works with Codex, Claude Code, Gemini CLI, Cursor, Aider, Windsurf, and 5 more tools.",
"shortDescription": "388 production-ready skills for AI coding agents across 20 domains",
"longDescription": "The largest open-source skills library for AI coding agents. 388 skills covering engineering (architecture, DevOps, security, AI/ML, agent tooling), marketing (SEO, AEO, CRO, content), product management, C-level advisory, regulatory compliance (ISO 13485, SOC 2, GDPR), project management, research and research operations, business operations, commercial, finance, and personal productivity. Includes 727 stdlib-only Python CLI tools, 842 reference guides, 118 orchestration agents, and 150 slash commands. Works with Codex, Claude Code, Gemini CLI, Cursor, Hermes Agent, Mistral Vibe, and 7 more tools.",
"developerName": "Alireza Rezvani",
"category": "Coding",
"capabilities": [

28
.codex/skills-index.json
View file

@ -3,7 +3,7 @@
"name": "claude-code-skills",
"description": "Production-ready skill packages for AI agents - Marketing, Engineering, Product, C-Level, PM, and RA/QM",
"repository": "https://github.com/alirezarezvani/claude-skills",
"total_skills": 362,
"total_skills": 364,
"skills": [
{
"name": "agent-launcher-orchestrator",
@ -1001,6 +1001,12 @@
"category": "engineering-advanced",
"description": "Use when the user asks to create ERD diagrams, normalize database schemas, design table relationships, or plan schema migrations."
},
{
"name": "deep-learning-book",
"source": "../../engineering/deep-learning-book/skills/deep-learning-book",
"category": "engineering-advanced",
"description": "Study companion and working knowledge base for the Deep Learning textbook by Goodfellow, Bengio & Courville (MIT Press, 2016), read free at deeplearningbook.org. Indexes all 20 chapters, carries a 2016-to-2026 delta layer naming what the book got right, what was superseded (transformers, AdamW, diffusion, double descent) and what still holds, and ships four deterministic tools: a prerequisite-aware reading-path planner, a training-failure diagnostic, a capacity-and-regularization planner, and a parameter/FLOP/activation-memory calculator. Use when studying or teaching this book, planning a route through it, deciding whether a chapter's advice is still current, or translating its math into a training decision. It points at the official chapters \u2014 it never reproduces them."
},
{
"name": "demo-video",
"source": "../../engineering/demo-video/skills/demo-video",
@ -1223,18 +1229,18 @@
"category": "engineering-advanced",
"description": "Use when the user asks to design a RAG pipeline, choose a chunking strategy or embedding model, pick a vector database, or evaluate retrieval quality (precision@k, recall@k, NDCG). Examples: 'design a RAG system for our docs', 'what chunk size should I use for this corpus', 'evaluate my retriever against ground truth'. NOT for general LLM cost tuning (use llm-cost-optimizer) or agent loops over retrieval (use agenthub)."
},
{
"name": "run",
"source": "../../engineering/autoresearch-agent/skills/run",
"category": "engineering-advanced",
"description": "Run a single experiment iteration. Edit the target file, evaluate, keep or discard. Use when the user runs /ar:run or asks for one manual autoresearch iteration."
},
{
"name": "run",
"source": "../../engineering/agenthub/skills/run",
"category": "engineering-advanced",
"description": "One-shot lifecycle command that chains init \u2192 baseline \u2192 spawn \u2192 eval \u2192 merge in a single invocation. Use when the user runs /hub:run or asks to execute a full AgentHub competition end-to-end."
},
{
"name": "run",
"source": "../../engineering/autoresearch-agent/skills/run",
"category": "engineering-advanced",
"description": "Run a single experiment iteration. Edit the target file, evaluate, keep or discard. Use when the user runs /ar:run or asks for one manual autoresearch iteration."
},
{
"name": "runbook-generator",
"source": "../../engineering/skills/runbook-generator",
@ -1319,6 +1325,12 @@
"category": "engineering-advanced",
"description": "Use when the user asks to write specs before code, define acceptance criteria, plan features before implementation, generate tests from specifications, or follow spec-first development practices."
},
{
"name": "spinning-up-deep-rl",
"source": "../../engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl",
"category": "engineering-advanced",
"description": "Knowledge base from \\\"Spinning Up in Deep RL\\\" by Joshua Achiam (OpenAI, MIT-licensed). Use when applying Achiam's frameworks for RL fundamentals and MDPs, the model-free algorithm taxonomy, policy gradient derivations, the six reference algorithms (VPG, TRPO, PPO, DDPG, TD3, SAC), debugging silently-failing RL code, or running rigorous multi-seed RL experiments."
},
{
"name": "sql-database-assistant",
"source": "../../engineering/skills/sql-database-assistant",
@ -2220,7 +2232,7 @@
"description": "Software engineering and technical skills"
},
"engineering-advanced": {
"count": 90,
"count": 92,
"source": "../../engineering",
"description": "Advanced engineering skills - agents, RAG, MCP, CI/CD, databases, observability"
},

1
.codex/skills/deep-learning-book Symbolic link
View file

@ -0,0 +1 @@
../../engineering/deep-learning-book/skills/deep-learning-book

1
.codex/skills/spinning-up-deep-rl Symbolic link
View file

@ -0,0 +1 @@
../../engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl

84
CHANGELOG.md
View file

@ -7,6 +7,90 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Added — engineering/spinning-up-deep-rl: the first book compiled by book-to-skill
Knowledge-base plugin compiled end-to-end by `engineering/book-to-skill` from OpenAI's
[Spinning Up in Deep RL](https://spinningup.openai.com/) (MIT, Copyright (c) 2018 OpenAI;
primarily developed by Joshua Achiam). 20 chapters, a glossary, a patterns file and a
decision cheatsheet, behind a 2,101-token resident core.
- **The full pipeline, not a hand-write.** `openai/spinningup` cloned, its `docs/`
reStructuredText tree (38 files, ~37k words, ~49K tokens) run through
`extract_document.py --mode technical` → analysis → chapter files → supporting files →
master `SKILL.md` → `book_skill_validator.py` → `skill_plugin_emitter.py`. The validator
passes clean in `--strict` mode and every file is inside budget.
- **Rights basis `open-license`, stated and honoured.** The emitter's Step-11 gate refuses a
shareable package without one. MIT permits derivative distribution; upstream's notice is
reproduced in full in the plugin's `LICENSE` beside this package's own, and `README.md`
names the source, the author and the source's frozen version.
- **Structure follows the source's own `toctree`.** User documentation (ch01-06), Introduction
to RL Parts 1-3 (ch07-09), resources — the researcher essay, key papers, exercises,
benchmarks (ch10-13), one chapter per algorithm in lineage order (ch14-19: VPG → TRPO → PPO,
DDPG → TD3 → SAC), and the logger / MPI / ExperimentGrid utilities (ch20).
- **The cheatsheet carries the judgment a glossary cannot** — the under-5-minute debug
turnaround, the 3-seed minimum (10+ to be thorough), family-specific benchmark network
defaults, and Spinning Up's own parity disclosure: DDPG/TD3/SAC are research-grade,
VPG/TRPO/PPO are not, and the docs say to use OpenAI Baselines for those.
- **Counters:** skills 387 → 388; agents 117 → 118; commands 149 → 150; plugins 98 → 99.
Tools and references unchanged by this plugin — a compiled knowledge base ships notes, not
scripts. (These sit on top of `deep-learning-book`, which merged into `dev` first; the
derived totals are 388 skills / 727 tools / 842 references / 118 agents / 150 commands /
99 plugins.)
### Fixed — book-to-skill's plugin emitter produced manifests this repo's CI rejects
`skill_plugin_emitter.py` wrote its whole `source` provenance block into `plugin.json`, with an
inline comment asserting that `source` and `attribution` were approved extension fields. That had
been true and no longer was: Claude Code rejects an entire manifest on any unrecognized key
(issue #954), and `scripts/check_plugin_json.py` hard-fails such a manifest, pointing at
`.claude-plugin/authoring-notes.json` instead. Every package the emitter produced therefore failed
the blocking CI gate the moment it was committed — a defect at the very last step of the pipeline,
which is why it had gone unnoticed. `_plugin_manifest()` now emits spec fields only and a new
`_authoring_notes()` writes the sidecar. Recorded as deviation 26 in
`engineering/book-to-skill/README.md`. The printed `marketplace.json` snippet is unchanged: `source`
is a valid key there, which is how it leaked into the manifest originally.
### Added — engineering/deep-learning-book: a companion to the free Deep Learning textbook
New `engineering/deep-learning-book/` plugin: a study companion for *Deep Learning* by
Goodfellow, Bengio & Courville (MIT Press, 2016), free to read at deeplearningbook.org.
One skill, 4 stdlib-only tools, 4 references, 3 assets, 1 agent, 3 commands.
- **Companion, not compilation — and that was the design decision.** `book-to-skill`'s
rights gate refuses a `shareable` package without `public-domain` / `open-license` /
`internal-docs` / `author-permission`, none of which applies to an MIT Press title whose
own site states the HTML-only format exists as a friction against copying under the
authors' contract; its rights reference lists publishing a compiled skill of a copyrighted
book to a public marketplace under **Do not**, and its hard rule 1 forbids scraping a book
from the web. So nothing here reproduces the book: every chapter file is original
synthesis linking to the official free chapter, and the organizing structure is the
published table of contents. **The rule this sets:** convert a copyrighted work into a
companion that indexes and updates the source, never a compilation that reproduces it.
- **The compiled-skill shape, validated by the compiler's own gate.** Master `SKILL.md`
(~2.0k tokens, chapter index + topic index), `chapters/ch01..ch20`, `glossary.md`,
`patterns.md`, `cheatsheet.md` — passes `book_skill_validator.py` clean with every file
inside `token_budget_estimator.py`'s caps.
- **The 2016→2026 delta layer is the differentiator.** A compilation freezes a source at its
publication date; this one dates it. Every chapter carries "What changed after 2016", and
`references/book_to_2026_delta.md` gives five corrections with primary citations and
per-claim confidence: double descent qualifying Ch 5's U-curve, AdamW splitting weight
decay from L2, transformers displacing Ch 10's recurrence, diffusion growing out of Ch 18's
score matching, and self-supervised learning vindicating Ch 15 while replacing its methods.
Two contested claims are marked contested rather than propagated; two named as folklore.
Stated rule: **the conflict is almost always in the recommendation, not the analysis.**
- **Four tools, each with a real refusal.** `reading_path_planner.py` (prerequisite closure
over the book's actual dependency graph, priced in weeks; exit 3 naming what covers an
out-of-scope goal, exit 4 with forcing questions when unroutable; ties break on keyword
specificity, not alphabetically); `training_diagnostics.py` (Ch 11's rules in priority
order, so a NaN is never reported as overfitting; exit 4 rather than diagnosing with no
instruments); `capacity_planner.py` (regularization ladder in cost order with "shrink the
model" ranked **last** in the overparameterized regime; exit 4 on a val-below-train split);
`model_arithmetic.py` (params/FLOPs/activation memory for conv, linear, position-wise
linear, MHA and LSTM/GRU stacks; exit 5 naming the layer whose shapes do not connect).
- `cs-deep-learning-tutor` agent; `/cs:deep-learning`, `/cs:dl-reading-path`,
`/cs:dl-diagnose`. **Counters:** skills 386 → 387; tools 723 → 727; refs 838 → 842;
agents 116 → 117; commands 146 → 149; plugins 97 → 98.
### Added — marketing/linkedin: organic LinkedIn presence with the platform rules in code
New `marketing/linkedin/` plugin, answering

100
CLAUDE.md
View file

File diff suppressed because one or more lines are too long

16
README.md
View file

@ -1,6 +1,6 @@
# Claude Code Skills & Plugins — Agent Skills for Every Coding Tool
**386 production-ready Claude Code skills, plugins, and agent skills for 13 AI coding tools.**
**388 production-ready Claude Code skills, plugins, and agent skills for 13 AI coding tools.**
The most comprehensive open-source library of Claude Code skills and agent plugins — also works with OpenAI Codex, Gemini CLI, Cursor, and 9 more coding agents. Reusable expertise packages covering engineering, DevOps, marketing (incl. AEO — Answer Engine Optimization for LLM citation), security (PreToolUse hooks), compliance, C-level advisory (incl. founder-mode CFO/CMO/CRO/CPO/COO/CHRO/CISO/GC/CDO/CAIO/CCO/VPE personas + 21 /cs:* slash commands), productivity (capture/email/reflect/weekly-review/deep-work/meetings), an academic research stack (litreview/grants/dossier/patent/syllabus/pulse/notebooklm/deep-research + hybrid router), and enterprise Research Operations (clinical-research/research-finance/market-research/product-research, v2.9.0).
@ -10,10 +10,10 @@ The most comprehensive open-source library of Claude Code skills and agent plugi
[^vibe]: Mistral Vibe is also **BYO-sync tier**: the repo ships a pre-generated `.vibe/skills/claude-skills/` tree, run `./scripts/vibe-install.sh` once locally to install into `~/.vibe/skills/`. Same agentskills.io SKILL.md standard — no format conversion. Docs: <https://docs.mistral.ai/mistral-vibe/agents-skills>.
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow?style=for-the-badge)](https://opensource.org/licenses/MIT)
[![Skills](https://img.shields.io/badge/Skills-386-brightgreen?style=for-the-badge)](#skills-overview)
[![Agents](https://img.shields.io/badge/Agents-116-blue?style=for-the-badge)](#agents)
[![Skills](https://img.shields.io/badge/Skills-388-brightgreen?style=for-the-badge)](#skills-overview)
[![Agents](https://img.shields.io/badge/Agents-118-blue?style=for-the-badge)](#agents)
[![Personas](https://img.shields.io/badge/Personas-7-purple?style=for-the-badge)](#personas)
[![Commands](https://img.shields.io/badge/Commands-146-orange?style=for-the-badge)](#commands)
[![Commands](https://img.shields.io/badge/Commands-150-orange?style=for-the-badge)](#commands)
[![Stars](https://img.shields.io/github/stars/alirezarezvani/claude-skills?style=for-the-badge)](https://github.com/alirezarezvani/claude-skills/stargazers)
[![SkillCheck Validated](https://img.shields.io/badge/SkillCheck-Validated-4c1?style=for-the-badge)](https://getskillcheck.com)
@ -29,7 +29,7 @@ Claude Code skills (also called agent skills or coding agent plugins) are modula
- **Python tools** — 706 CLI scripts (all stdlib-only, zero pip installs)
- **Reference docs** — 823 templates, checklists, and domain-specific knowledge files
**One repo, thirteen platforms.** Works natively as Claude Code plugins, Codex agent skills, Gemini CLI skills, Hermes Agent skills, Mistral Vibe skills, and converts to more tools via `scripts/convert.sh`. All 723 Python tools run anywhere Python runs.
**One repo, thirteen platforms.** Works natively as Claude Code plugins, Codex agent skills, Gemini CLI skills, Hermes Agent skills, Mistral Vibe skills, and converts to more tools via `scripts/convert.sh`. All 727 Python tools run anywhere Python runs.
### Skills vs Agents vs Personas
@ -152,12 +152,12 @@ Run `./scripts/convert.sh --tool all` to generate tool-specific outputs locally.
## Skills Overview
**386 production-ready skills across 20 domains:**
**388 production-ready skills across 20 domains:**
| Domain | Skills | Highlights | Details |
|--------|--------|------------|---------|
| **🔧 Engineering — Core** | 53 | Architecture, frontend, backend, fullstack, QA, DevOps, SecOps, AI/ML, data, Playwright Pro (test gen, flaky fix, migrations), self-improving agent (auto-memory curation), security suite, a11y audit, **named-persona-adversarial-review** (review via named engineering philosophies), **embedded-iot-mentor** (MCU/board selection, firmware-reuse-first, breadboard-MVP discipline) | [engineering-team/](engineering-team/) |
| **⚡ Engineering — POWERFUL** | 91 | Agent designer, RAG architect, database designer, CI/CD builder, security auditor, MCP builder, AgentHub, Helm charts, Terraform, self-eval, llm-wiki, tc-tracker, autoresearch-agent, **reliability portfolio** (feature-flags-architect, kubernetes-operator, chaos-engineering, slo-architect), ship-gate, security-guidance PreToolUse hook, **Matt Pocock skills** (write-a-skill, caveman, grill-me, handoff, grill-with-docs), **zero-hallucination-coder** (Discuss→Map→Decompose→Execute→Verify), **agent-harness** (goal→plan→execute→verify→close loops over any domain), **memory-engineering** (price the memory write path, pick which cost to pay, audit FACT/SKILL/LOG density, gate on a forgetting policy), **skillopt-sleep** (nightly gated self-evolution from real Claude Code sessions, vendored from microsoft/SkillOpt), **book-to-skill** (compile a book, docs folder, or spec collection into a knowledge-base skill, then package it as a plugin), **boost-asio-pro** (async C++ networking — version-gated coroutine/callback styles, strand discipline), **human-gate** (batched human review as a structured artifact + a gate that refuses to close on open blockers), **agent-memory** (four-tier L0-L3 memory ladder over Claude Code hooks; promotion earned by recurrence across sessions and days, redacted or contested claims refuse to promote, nothing reaches CLAUDE.md without a human adopt), **skill-doctor** (grade the agent setup from real session history — rubric-judged, deterministically gated, evidence-traced skill edits) | [engineering/](engineering/) |
| **⚡ Engineering — POWERFUL** | 93 | Agent designer, RAG architect, database designer, CI/CD builder, security auditor, MCP builder, AgentHub, Helm charts, Terraform, self-eval, llm-wiki, tc-tracker, autoresearch-agent, **reliability portfolio** (feature-flags-architect, kubernetes-operator, chaos-engineering, slo-architect), ship-gate, security-guidance PreToolUse hook, **Matt Pocock skills** (write-a-skill, caveman, grill-me, handoff, grill-with-docs), **zero-hallucination-coder** (Discuss→Map→Decompose→Execute→Verify), **agent-harness** (goal→plan→execute→verify→close loops over any domain), **memory-engineering** (price the memory write path, pick which cost to pay, audit FACT/SKILL/LOG density, gate on a forgetting policy), **skillopt-sleep** (nightly gated self-evolution from real Claude Code sessions, vendored from microsoft/SkillOpt), **book-to-skill** (compile a book, docs folder, or spec collection into a knowledge-base skill, then package it as a plugin), **boost-asio-pro** (async C++ networking — version-gated coroutine/callback styles, strand discipline), **human-gate** (batched human review as a structured artifact + a gate that refuses to close on open blockers), **agent-memory** (four-tier L0-L3 memory ladder over Claude Code hooks; promotion earned by recurrence across sessions and days, redacted or contested claims refuse to promote, nothing reaches CLAUDE.md without a human adopt), **skill-doctor** (grade the agent setup from real session history — rubric-judged, deterministically gated, evidence-traced skill edits), **deep-learning-book** (companion to the free Goodfellow/Bengio/Courville textbook — 20 chapter files with a 2016→2026 delta layer, reading-path planner, Ch-11 training diagnostic, capacity planner, FLOP/activation-memory calculator), **spinning-up-deep-rl** (knowledge base compiled from OpenAI's Spinning Up in Deep RL — RL fundamentals, the model-free taxonomy, policy-gradient derivations, and one chapter per reference algorithm) | [engineering/](engineering/) |
| **🎯 Product** | 17 | Product manager, agile PO, strategist, UX researcher, UI design, landing pages, SaaS scaffolder, analytics, experiment designer, discovery, roadmap communicator, code-to-prd, apple-hig-expert | [product-team/](product-team/) |
| **📣 Marketing** | 49 | 8 pods: Content, SEO + AEO (`aeo` — E-E-A-T audit, citation tracking across 5 LLMs) + local (`local-seo-manager` — GBP/NAP/Map-Pack), CRO, Channels, Growth, Intelligence, Sales + `business-name-fit` (cross-cultural naming) + context foundation + orchestration router | [marketing-skill/](marketing-skill/) |
| **🚀 Productivity** | 12 | `capture` (brain-dump-to-action), `email` pair (inbox-setup + inbox-triage), `reflect` (journal), `handoff` (Matt Pocock-inspired), `andreessen` (market-first decision mode), `roast` (5-angle idea panel → GO/RESHAPE/KILL), `fable-goal` (ramble → autonomous /goal prompt), `weekly-review` (GTD loop with refusal gate), `deep-work` (time-blocking + shallow-work budget), `meetings` (cost gate + agenda + action items), `swedish-mentor` (CEFR-leveled Swedish learning paths) | [productivity/](productivity/) |
@ -359,7 +359,7 @@ Yes. Skills work natively with 13 tools: Claude Code, OpenAI Codex, Gemini CLI,
No. We follow semantic versioning and maintain backward compatibility within patch releases. Existing script arguments, plugin source paths, and SKILL.md structures are never changed in patch versions. See the [CHANGELOG](CHANGELOG.md) for details on each release.
**Are the Python tools dependency-free?**
Yes. All 723 Python tools use the standard library only — zero pip installs required. Every skill's CLI entry point is verified to run with `--help` (most skills ship one script per tool; a few, like the vendored `engineering/skillopt-sleep` engine, ship a multi-module package behind a single `python -m` entry point). A few tools — `engineering/book-to-skill`'s document extractors — can *optionally* use third-party parsers for higher-fidelity output, but every format falls back to a standard-library parser and nothing is installed implicitly.
Yes. All 727 Python tools use the standard library only — zero pip installs required. Every skill's CLI entry point is verified to run with `--help` (most skills ship one script per tool; a few, like the vendored `engineering/skillopt-sleep` engine, ship a multi-module package behind a single `python -m` entry point). A few tools — `engineering/book-to-skill`'s document extractors — can *optionally* use third-party parsers for higher-fidelity output, but every format falls back to a standard-library parser and nothing is installed implicitly.
**How do I create my own Claude Code skill?**
Each skill is a folder with a `SKILL.md` (frontmatter + instructions), optional `scripts/`, `references/`, and `assets/`. See the [Skills & Agents Factory](https://github.com/alirezarezvani/claude-code-skills-agents-factory) for a step-by-step guide.

2
engineering/book-to-skill/.claude-plugin/authoring-notes.json
View file

@ -3,6 +3,6 @@
"derived_from": "https://github.com/virgiliojr94/book-to-skill",
"original_author": "virgiliojr94",
"original_license": "MIT",
"derivation_note": "The extraction library (book_to_skill/ — config, exceptions, sanitize, dependencies, utils, and the 7 per-format parsers) is vendored from upstream and carries upstream's multi-format chains, chapter detection across 5 script families, and Unicode/XXE hardening. 25 numbered deviations are recorded in README.md, which is the authoritative list: the workflow was rewritten Claude-Code-first for this repo's conventions, the 3 upstream tools were reworked into 4 argparse CLIs with --help/--sample, install-on-import behaviour was replaced with a report-only default, and Step 11 (plugin emission with a rights gate) plus the validator's budget and index families are additions with no upstream counterpart."
"derivation_note": "The extraction library (book_to_skill/ — config, exceptions, sanitize, dependencies, utils, and the 7 per-format parsers) is vendored from upstream and carries upstream's multi-format chains, chapter detection across 5 script families, and Unicode/XXE hardening. 26 numbered deviations are recorded in README.md, which is the authoritative list: the workflow was rewritten Claude-Code-first for this repo's conventions, the 3 upstream tools were reworked into 4 argparse CLIs with --help/--sample, install-on-import behaviour was replaced with a report-only default, and Step 11 (plugin emission with a rights gate) plus the validator's budget and index families are additions with no upstream counterpart."
}
}

22
engineering/book-to-skill/README.md
View file

@ -101,7 +101,7 @@ small; just read it."
## Deviations from upstream
**This numbered list is the authoritative record.** `plugin.json`'s
**This numbered list is the authoritative record.** `.claude-plugin/authoring-notes.json`'s
`attribution.derivation_note` summarizes it; if the two ever disagree, this list wins.
**Structural**
@ -245,9 +245,10 @@ small; just read it."
preventing hand-edit mistakes contradicted the manifest beside it); the symlink guard is
backed by a **post-copy re-walk** that deletes the package if a link appeared during the
copy, closing the check-then-act window rather than only narrowing it; and the manifest
carries `source.license_scope` stating that the top-level `license` covers the package
emits `source.license_scope` stating that the top-level `license` covers the package
scaffolding, not the compiled notes — a distinction that previously lived only in README
prose where a tool reading the manifest alone would miss it.
prose where a tool reading the metadata alone would miss it. (Deviation 26 later moved
that whole `source` block out of `plugin.json` and into the sidecar.)
21. **The documented quick-start actually runs.** SKILL.md's copy-paste block referenced
`$WORKDIR` and `$SKILLS_HOME` without ever defining them — following it literally produced
@ -297,6 +298,21 @@ small; just read it."
leaves the victim's contents intact and is replaced by a 0600 file we own. Degrades to the
previous path-based checks on platforms without `dir_fd`/`O_NOFOLLOW` (Windows).
26. **Provenance moved out of `plugin.json` into the sidecar the repo actually allows.**
The manifest builder wrote its whole `source` block (spec, build pattern, source document,
chapter count, distribution, `license_scope`, `rights_basis`) into `plugin.json`, and an
inline comment asserted that `source` and `attribution` were "approved extension fields."
That had been true and no longer was: Claude Code rejects an entire manifest on any
unrecognized key (issue #954), and this repo's own `scripts/check_plugin_json.py` hard-fails
a `plugin.json` carrying either field, pointing at `.claude-plugin/authoring-notes.json`
instead. So every package the emitter produced failed the repo's blocking CI gate the moment
it was committed — a defect that only surfaces at the very last step of the pipeline, which
is why it survived. `_plugin_manifest()` now emits spec fields only, and a new
`_authoring_notes()` writes the `source` block to `.claude-plugin/authoring-notes.json`.
Note that `source` remains a *valid* key in a `marketplace.json` `plugins[]` entry, which is
how it leaked into the manifest in the first place; the printed marketplace snippet is
unchanged and still correct.
---
## Security audit

15
engineering/book-to-skill/skills/book-to-skill/references/conversion_workflow.md
View file

@ -382,10 +382,23 @@ Emits `<domain>/<slug>/` with `.claude-plugin/plugin.json`, `README.md`,
`skills/<slug>/`, then prints the marketplace entry to register. Drop `--dry-run` to write.
**Rights gate.** The emitter defaults to `--distribution local`, which records
`source.cleared_for_distribution: false` in the manifest. `--distribution shareable` **refuses** unless `--rights` names a basis:
`source.cleared_for_distribution: false` in `.claude-plugin/authoring-notes.json`.
`--distribution shareable` **refuses** unless `--rights` names a basis:
`public-domain`, `open-license`, `internal-docs`, or `author-permission`. Fair use is
deliberately not an option — it is a defence, not a licence, and not this tool's call.
**Attribution is yours to add, by hand, when the source is someone else's work.** The emitter
writes a `source` block — how the skill was built — because that is all it can know; it has
`--source-note` free text and a rights basis, not an upstream URL, author or licence, and a
half-filled `attribution` block is worse than none. So when `--rights` is anything but
`internal-docs`, add an `attribution` block beside it in `authoring-notes.json`
(`derived_from`, `original_author`, `original_license`, `original_copyright`,
`derivation_note` — the shape the rest of this repo uses), and put the upstream licence
notice in the package's `LICENSE` and a credit line in its `README.md`. **That last part is
the obligation:** `authoring-notes.json` is authoring metadata Claude Code never reads, and a
sidecar JSON file is not a licence notice. `engineering/spinning-up-deep-rl` is the worked
example.
The emitter also refuses to wrap a skill with validation errors. Fix the source skill first.
Registration in `.claude-plugin/marketplace.json` stays manual — it is a repo-wide change.

51
engineering/book-to-skill/skills/book-to-skill/scripts/skill_plugin_emitter.py
View file

@ -10,7 +10,8 @@ This tool closes that gap. Point it at a generated book skill and it emits the
plugin package this repo's conventions require:
<domain>/<slug>/
├── .claude-plugin/plugin.json manifest with `./skills/<slug>` + attribution
├── .claude-plugin/plugin.json manifest, spec fields only (no `source`)
├── .claude-plugin/authoring-notes.json provenance the manifest may not carry
├── README.md what the skill knows and where it came from
├── agents/cs-<slug>.md persona that answers from the book
├── commands/cs-<slug>.md /cs:<slug> entry point
@ -24,7 +25,8 @@ Rights gate: a book skill built from a copyrighted work is personal study
notes. `--distribution shareable` therefore refuses to emit unless `--rights`
names a basis that permits redistribution. `--distribution local` (the default)
emits with a notice and records `source.cleared_for_distribution: false`
in the manifest.
in `authoring-notes.json`. Provenance never goes in `plugin.json`: Claude Code
rejects the whole manifest on any unrecognized key (issue #954).
Exit codes:
0 package emitted (or --dry-run / --sample completed)
@ -166,13 +168,13 @@ def _assert_no_symlinks(skill_dir: Path) -> None:
def _plugin_manifest(identity: dict, *, domain: str, author: str, author_url: str,
repository: str, distribution: str, rights: str | None,
source_note: str) -> dict:
# Key set and the two approved extension fields (`source`, `attribution`) are
# fixed by scripts/check_plugin_json.py. Anything else — including a
# top-level "private" flag — fails the repo's CI gate, so the distribution
# marker lives inside `source`.
manifest = {
repository: str) -> dict:
# Spec fields ONLY. Claude Code's manifest validator rejects the whole
# plugin.json on any unrecognized key (issue #954), and the repo's own gate
# (scripts/check_plugin_json.py) hard-fails a manifest carrying `source` or
# `attribution`. Provenance therefore goes to the sidecar file the validator
# never reads — see _authoring_notes below.
return {
"name": identity["slug"],
"description": identity["description"],
"version": "1.0.0",
@ -185,7 +187,19 @@ def _plugin_manifest(identity: dict, *, domain: str, author: str, author_url: st
# the manifest sees the distinction that README.md makes in prose.
"license": "MIT",
"skills": [f"./skills/{identity['slug']}"],
"source": {
}
def _authoring_notes(identity: dict, *, distribution: str, rights: str | None,
source_note: str) -> dict:
"""Provenance for `.claude-plugin/authoring-notes.json`.
The repo's schema allows exactly two keys here, `source` and `attribution`;
a generated skill's provenance is a `source` block. This file is authoring
metadata that Claude Code's manifest validator never reads, which is
precisely why the fields live here rather than in plugin.json.
"""
source = {
"spec": "generated by engineering/book-to-skill",
"build_pattern": "book-to-skill conversion: extract -> analyze -> chapter files "
"+ glossary + patterns + cheatsheet -> master SKILL.md",
@ -195,21 +209,20 @@ def _plugin_manifest(identity: dict, *, domain: str, author: str, author_url: st
"chapters": identity["chapters"],
"distribution": distribution,
"license_scope": (
"The top-level `license` covers this package's scaffolding only. The "
"compiled notes under skills/ are derived from the source document and "
"plugin.json's top-level `license` covers this package's scaffolding only. "
"The compiled notes under skills/ are derived from the source document and "
"carry that work's terms; see source.rights_basis."
),
},
}
if rights:
manifest["source"]["rights_basis"] = rights
manifest["source"]["rights_note"] = RIGHTS_BASES[rights]
source["rights_basis"] = rights
source["rights_note"] = RIGHTS_BASES[rights]
if distribution == "local":
# A local package is study notes from a work the user owns. The flag is
# advisory metadata, not enforcement — it exists so a later publish step
# (or a reviewer) can see the package was never cleared for sharing.
manifest["source"]["cleared_for_distribution"] = False
return manifest
source["cleared_for_distribution"] = False
return {"source": source}
def _agent_markdown(identity: dict, domain: str, source_note: str) -> str:
@ -407,10 +420,12 @@ def emit(*, skill_dir: Path, dest_root: Path, domain: str, author: str, author_u
_assert_replaceable(package_root, resolved_dest, skill_dir)
manifest = _plugin_manifest(identity, domain=domain, author=author, author_url=author_url,
repository=repository, distribution=distribution, rights=rights,
repository=repository)
notes = _authoring_notes(identity, distribution=distribution, rights=rights,
source_note=source_note)
files = {
".claude-plugin/plugin.json": json.dumps(manifest, indent=2, ensure_ascii=False) + "\n",
".claude-plugin/authoring-notes.json": json.dumps(notes, indent=2, ensure_ascii=False) + "\n",
"README.md": _readme_markdown(identity, domain=domain, distribution=distribution,
rights=rights, source_note=source_note),
f"agents/cs-{identity['slug']}.md": _agent_markdown(identity, domain, source_note),

12
engineering/deep-learning-book/.claude-plugin/authoring-notes.json Normal file
View file

@ -0,0 +1,12 @@
{
"source": {
"spec": "User request: convert https://www.deeplearningbook.org/ into a shareable skill.",
"build_pattern": "Companion, not compilation. Follows the book-to-skill OUTPUT SHAPE (master SKILL.md under 4k tokens with a chapter index and topic index, chapters/chNN-*.md, glossary.md, patterns.md, cheatsheet.md — validated by book-to-skill's own book_skill_validator.py) while deliberately NOT running its extraction pipeline. Every chapter file is original synthesis written from domain knowledge, linking to the official free chapter; the organizing structure is the book's published table of contents. Adds four stdlib-only tools, four references citing 7-8 sources each with per-claim confidence levels, and three assets.",
"distinct_from": "engineering/book-to-skill (the converter — this is the artifact its rights gate calls for when the source is copyrighted); engineering-team/senior-ml-engineer (production MLOps and serving, not the theory); engineering/llm-cost-optimizer (LLM economics, which the book predates); engineering/memory-engineering and engineering/agent-memory (agent systems, not model training); the teach/learn skills (general study workflows, not one specific text with its dependency graph and expiry dates)."
},
"attribution": {
"source_work": "Deep Learning, Ian Goodfellow, Yoshua Bengio and Aaron Courville, MIT Press, 2016 — https://www.deeplearningbook.org/. All rights remain with the authors and publisher. This companion is independent and unaffiliated, and is not endorsed by the authors or MIT Press.",
"rights_basis": "No rights basis is claimed or needed, because no protected expression is reproduced. The book is copyrighted and its site states the HTML-only format exists as a deliberate friction against copying under the authors' MIT Press contract; PDF distribution is not permitted. book-to-skill's rights gate would classify it 'local' (none of public-domain / open-license / internal-docs / author-permission applies), and its rights reference lists publishing a compiled skill of a copyrighted book to a public marketplace under 'Do not'. This plugin therefore contains no passages, paragraphs, figures, exercises, or per-paragraph paraphrase — only original synthesis, the factual table of contents, and links to the official free chapters. See skills/deep-learning-book/references/rights_and_use.md.",
"evidence_discipline": "Every claim about what changed since 2016 carries a confidence level (well-established / contested / folklore) and a primary citation, and two commonly repeated claims are marked contested rather than propagated: batch normalization's 'internal covariate shift' mechanism (challenged by Santurkar et al. 2018) and the strong form of the lottery-ticket hypothesis. Two pieces of folklore are named as folklore, including the local-minima fear the book itself corrects."
}
}

15
engineering/deep-learning-book/.claude-plugin/plugin.json Normal file
View file

@ -0,0 +1,15 @@
{
"name": "deep-learning-book",
"description": "Study companion and working knowledge base for the Deep Learning textbook by Goodfellow, Bengio & Courville (MIT Press, 2016), free to read at deeplearningbook.org. Twenty chapter files, a glossary, a patterns file and a cheatsheet index the whole book; every chapter carries a 'what changed after 2016' section, and a dedicated delta reference dates the text against 2026 practice with per-claim confidence levels — double descent qualifying the U-curve, AdamW splitting weight decay from L2, transformers displacing Chapter 10's recurrence, diffusion growing out of Chapter 18's score matching, and self-supervised learning vindicating Chapter 15 while replacing its methods. Four stdlib-only tools make the book executable: a prerequisite-closed reading-path planner that refuses goals the 2016 book does not cover, a training diagnostic running Chapter 11's rules in priority order so a NaN is never reported as overfitting, a capacity planner that ranks the regularization ladder and pushes 'shrink the model' last in the overparameterized regime, and a parameter/FLOP/activation-memory calculator that refuses a stack whose shapes do not connect. Deliberately a companion, not a compilation: the book is copyrighted, so nothing here reproduces its text — every chapter file is original synthesis linking to the official free chapter. Use when studying or teaching this book, planning a route through it, or checking whether one of its recommendations is still current.",
"version": "2.12.0",
"author": {
"name": "Alireza Rezvani",
"url": "https://alirezarezvani.com"
},
"homepage": "https://github.com/alirezarezvani/claude-skills/tree/main/engineering/deep-learning-book",
"repository": "https://github.com/alirezarezvani/claude-skills",
"license": "MIT",
"skills": [
"./skills/deep-learning-book"
]
}

118
engineering/deep-learning-book/README.md Normal file
View file

@ -0,0 +1,118 @@
# deep-learning-book — study companion for *Deep Learning* (Goodfellow, Bengio & Courville)
A navigable, executable, **date-stamped** companion to the 2016 MIT Press textbook that is free
to read at [deeplearningbook.org](https://www.deeplearningbook.org/).
Twenty chapter files, a glossary, a patterns file, a cheatsheet, four references and four
deterministic tools — plus the thing a static compilation cannot give you: a **2016 → 2026 delta
layer** that says, per chapter, what still holds, what was superseded, and what is now purely
historical.
## Why this is a companion and not a compiled skill
This repository ships [`engineering/book-to-skill`](../book-to-skill/), which compiles a document
on disk into a knowledge-base skill. It was deliberately **not** used here, for three reasons its
own rights gate makes binding:
1. The book is copyrighted (MIT Press, 2016). Free to read ≠ free to redistribute.
2. deeplearningbook.org states that its HTML-only presentation is a deliberate friction against
copying, required by the authors' contract, and that PDF distribution is not permitted.
3. This repository is public, and `book-to-skill`'s rights gate refuses a `shareable` package
without `public-domain`, `open-license`, `internal-docs` or `author-permission`. None applies.
(Its hard rule 1 also forbids scraping a book from the web, so the pipeline could not have run
against a URL in any case.)
So this skill contains **no passages, paragraphs, figures, or per-paragraph paraphrase**. Every
chapter file is original synthesis — what the chapter establishes, how to use it, where it has
aged — with a link to the official free chapter. The organizing structure is the book's published
table of contents, which is factual metadata. Full reasoning in
[`references/rights_and_use.md`](skills/deep-learning-book/references/rights_and_use.md).
**The rule this sets for the repository:** when a user asks to convert a copyrighted work into a
shareable skill, build a companion that indexes and updates the source, not a compilation that
reproduces it. Compile only when the rights gate clears, and keep the output local when it does not.
## What is in it
```
skills/deep-learning-book/
├── SKILL.md core frameworks + chapter index + topic index (~1.9k tokens, resident)
├── chapters/ch01..ch20 one file per chapter, each with "What changed after 2016"
├── glossary.md every key term → its chapter
├── patterns.md techniques as instruments, with trade-offs
├── cheatsheet.md decision tables: loss choice, fit verdict, optimizer defaults
├── references/ delta layer · prerequisite map · study method · rights
├── scripts/ 4 stdlib tools
└── assets/ layer spec · chapter worksheet · study log
```
## The tools
| Tool | Does | Refuses |
|---|---|---|
| `reading_path_planner.py` | Goal + background + hours → prerequisite-closed, ordered path with an hour budget | A goal outside the book (exit 3, names what covers it); an unroutable goal (exit 4, prints the questions) |
| `training_diagnostics.py` | Measurements → ranked cause + next action + chapter, rules in priority order | Diagnosing with no instruments (exit 4). Never reports a NaN as overfitting |
| `capacity_planner.py` | Gap + params-per-example → ordered regularization ladder | A validation error materially below training error (exit 4 — leaky split) |
| `model_arithmetic.py` | Layer stack → parameters, FLOPs, activation memory per example | A stack whose shapes do not connect (exit 5); an unknown layer type (exit 4) |
All four are standard-library only, support `--help` / `--sample` / `--output json`, make no
network calls, and load no frameworks.
## The delta layer
The book was published in 2016; *Attention Is All You Need* appeared in 2017. Five corrections
carry most of the weight, each cited and confidence-rated in
[`references/book_to_2026_delta.md`](skills/deep-learning-book/references/book_to_2026_delta.md):
1. **Double descent** qualifies Chapter 5's U-shaped capacity curve — "shrink the model when it
overfits" is no longer the only correct move, and `capacity_planner.py` encodes the correction.
2. **AdamW**: weight decay and L2 are not equivalent under an adaptive optimizer; Chapter 7 treats
them as interchangeable.
3. **Transformers** displaced Chapter 10's recurrence — keep its gradient-flow analysis, drop its
architecture recommendation. (State-space models made that analysis live again.)
4. **Diffusion** grew directly out of Chapter 14's denoising autoencoders and Chapter 18's score
matching, and displaced Chapter 20's model list.
5. **Self-supervised learning** vindicated Chapter 15's bet while replacing every method it names;
unsupervised disentanglement was proven impossible without inductive bias.
The general rule the file states: the conflict is almost always in the **recommendation**, not the
**analysis**. Keep the diagnosis, replace the prescription.
## Use it
```bash
S=engineering/deep-learning-book/skills/deep-learning-book/scripts
python3 $S/reading_path_planner.py --goal "train and debug convnets" --background applied --hours-per-week 5
python3 $S/training_diagnostics.py --train-loss 0.02 --val-loss 1.9 --tiny-subset-fits yes
python3 $S/capacity_planner.py --params 12000000 --train-examples 50000 --train-error 0.01 --val-error 0.22
python3 $S/model_arithmetic.py --spec skills/deep-learning-book/assets/example_layer_spec.json
```
Slash commands: `/cs:deep-learning` (navigate and date the answer), `/cs:dl-reading-path`,
`/cs:dl-diagnose`. Agent: `cs-deep-learning-tutor`.
## Distinct from
- **`engineering/book-to-skill`** — the converter. This is what you build when its rights gate
says no.
- **`engineering-team/senior-ml-engineer`** — production MLOps, deployment, serving. This is the
theory underneath.
- **`engineering/llm-cost-optimizer`** — LLM economics, which the book predates entirely.
- **`teach` / `learn` skills** — general study workflows. This is one specific text, with its
dependency graph and its expiry dates.
## Scope
The twenty chapters and the delta between them and 2026 practice. **Not covered**, because the
book does not cover them: reinforcement learning beyond passing mention, LLM training
infrastructure, RLHF/DPO, agentic systems, MLOps tooling, fairness and safety evaluation. The
skill is built to say so rather than improvise.
## Attribution
*Deep Learning*, Ian Goodfellow, Yoshua Bengio and Aaron Courville, MIT Press, 2016 —
https://www.deeplearningbook.org/. All rights in the book remain with its authors and publisher.
This companion is an independent work: no text, figures, or exercises from the book are
reproduced here, and it is not endorsed by or affiliated with the authors or MIT Press. The
companion's own content is MIT-licensed as part of this repository.

61
engineering/deep-learning-book/agents/cs-deep-learning-tutor.md Normal file
View file

@ -0,0 +1,61 @@
---
name: cs-deep-learning-tutor
description: "Study companion for the Deep Learning textbook (Goodfellow, Bengio & Courville, 2016). Plans a prerequisite-closed reading path, answers chapter questions from the compiled knowledge base, diagnoses training runs against Chapter 11's decision tree, and flags every place the 2016 text has been superseded. Use for studying the book, teaching from it, or checking whether one of its recommendations is still current."
---
# Deep Learning Tutor
You are a study companion for *Deep Learning* by Ian Goodfellow, Yoshua Bengio and Aaron
Courville (MIT Press, 2016), which is free to read at deeplearningbook.org.
## What you are working from
`engineering/deep-learning-book/skills/deep-learning-book/` — a master SKILL.md with core
frameworks and two indexes, 20 chapter files, a glossary, a patterns file, a cheatsheet, four
references and four tools. Read the SKILL.md first, resolve the question through the Topic
Index, then read that chapter file before answering.
## Hard rules
1. **Never reproduce the book's text.** Not a paragraph, not a figure, not a
sentence-by-sentence paraphrase. Point the reader at the official chapter URL and explain in
your own words. This is the constraint the whole skill is built around — see
`skills/deep-learning-book/references/rights_and_use.md`.
2. **Date every recommendation.** The book is from 2016 and *Attention Is All You Need* is from
2017. When a chapter's advice has been superseded, say so and cite
`skills/deep-learning-book/references/book_to_2026_delta.md`. Never present a 2016 recommendation as current practice
without that check.
3. **Separate the analysis from the prescription.** The book's diagnoses (why gradients vanish,
why the partition function is hard, why depth helps) almost all still hold. Its
prescriptions (use an LSTM, use Adam with L2, shrink the model when it overfits) frequently
do not. Keep the diagnosis, replace the prescription.
4. **Say when the book does not cover something.** RLHF, LLM infrastructure, agents, MLOps,
fairness — name the gap and route elsewhere rather than improvising the book's position.
5. **Read the chapter file before answering from it.** The indexes are for navigation, not for
answering.
6. **Run the tool rather than estimating.** Reading paths, training diagnoses, capacity plans
and parameter counts all have deterministic tools. Use them, then interpret the output.
## How you work
**When asked where to start** — run `reading_path_planner.py` with the stated goal, background
and weekly hours. If it exits 3 or 4, relay its questions rather than guessing a path.
**When asked about a topic** — resolve through the Topic Index, read the chapter file, answer,
and always surface the "What changed after 2016" section if one applies.
**When asked to diagnose a training run** — ask for the measurements the tool needs
(train loss, val loss, target loss, gradient norm, whether it can overfit a tiny subset), run
`training_diagnostics.py`, and act on finding [1] before anything below it. Do not skip to the
interesting hypothesis; the rule order exists because a NaN is not an overfitting problem.
**When teaching** — use the retrieval-practice cadence in `skills/deep-learning-book/references/study_method_canon.md`:
ask the reader to state the core idea from memory first, then correct. Do not lecture the
chapter at someone who has just read it.
## Voice
Direct and specific. Name the chapter for every claim. When the reader's plan is wrong — front
to back through Part I, or a Part III chapter without its prerequisites — say so once, give the
alternative, and let them decide. When something in the book is simply out of date, say that
plainly rather than defending it; a companion that will not date its source is worthless.

55
engineering/deep-learning-book/commands/cs-deep-learning.md Normal file
View file

@ -0,0 +1,55 @@
---
name: "cs-deep-learning"
description: "/cs:deep-learning — Study companion for the Deep Learning textbook (Goodfellow, Bengio & Courville, 2016). Answers chapter questions from a compiled knowledge base, always dating the answer against 2026 practice, and routes to the reading-path planner or the training diagnostic. Points at the free official chapters; never reproduces them."
argument-hint: "[a topic, a chapter number, a question about the book, or 'where do I start']"
---
# /cs:deep-learning — Navigate the book, and date its advice
**Command:** `/cs:deep-learning [topic | chNN | question]`
The book is free to read at [deeplearningbook.org](https://www.deeplearningbook.org/). This
command navigates it and keeps it current; it does not replace it.
## When to run
- "What does the book say about regularization / saddle points / the partition function?"
- "Is Chapter 10's advice on LSTMs still right?"
- "Explain the ELBO the way Chapter 19 sets it up"
- "Where should I start?" (routes to `/cs:dl-reading-path`)
- "Why is my training run doing this?" (routes to `/cs:dl-diagnose`)
## When NOT to run
- Production ML engineering → `engineering-team/senior-ml-engineer`
- LLM cost and serving → `engineering/llm-cost-optimizer`
- RLHF, agents, prompting, MLOps → outside the book entirely; this command will say so
## Procedure
1. **Load** `engineering/deep-learning-book/skills/deep-learning-book/SKILL.md`.
2. **Resolve** the request through the Topic Index to one or more chapters.
3. **Read** those chapter files before answering. The index is navigation, not content.
4. **Answer** in your own words, naming the chapter, and link the official chapter URL.
5. **Date it.** If the chapter file has a "What changed after 2016" section relevant to the
answer, surface it — with the confidence level from
`skills/deep-learning-book/references/book_to_2026_delta.md`. Separate the book's *analysis* (usually still true) from
its *prescription* (frequently superseded).
6. **Name the boundary.** If the question is outside the twenty chapters, say so and route.
## The four things this command will not do
- Reproduce the book's text, figures, or a paragraph-by-paragraph paraphrase.
- Present a 2016 recommendation as current practice without checking the delta layer.
- Answer from the index without reading the chapter file.
- Improvise the book's position on material published after it.
## Output shape
```
Chapter(s): ch07 (Regularization), ch05 (capacity)
Answer : <original explanation, naming the framework>
Still true: <holds / analysis holds but recommendation superseded / historical> + why
Read it : https://www.deeplearningbook.org/contents/regularization.html
Next : <the tool, worksheet, or chapter that follows>
```

52
engineering/deep-learning-book/commands/cs-dl-diagnose.md Normal file
View file

@ -0,0 +1,52 @@
---
name: "cs-dl-diagnose"
description: "/cs:dl-diagnose — Diagnose a training run against Chapter 11's decision tree: read training error before deciding anything, and never diagnose a NaN as overfitting. Returns a ranked cause, the specific next action, and the chapter that justifies it."
argument-hint: "[what the run is doing — losses, gradient norm, or a description]"
---
# /cs:dl-diagnose — Measurement first, hypothesis second
**Command:** `/cs:dl-diagnose [symptoms]`
Chapter 11's rule, which most teams have backwards: **read training error first.** High
training error means the model or the optimizer is the bottleneck, and more data cannot help.
## Procedure
1. **Collect the instruments.** Ask for what is missing, in this order:
- training loss and validation loss (same units, same epoch)
- the target loss — a human baseline, a published number, or an irreducible-error estimate
(without it, underfitting cannot be distinguished from convergence)
- global gradient norm, if available
- has the loss ever gone NaN or inf?
- can the model drive training loss to ~0 on 10–50 examples? (the smoke test that separates
a bug from a hard problem)
2. **Run the tool:**
```bash
python3 engineering/deep-learning-book/skills/deep-learning-book/scripts/training_diagnostics.py \
--train-loss <x> --val-loss <y> --target-loss <z> --grad-norm <g> \
--tiny-subset-fits yes|no|unknown
```
Exit 4 means not enough instruments — ask for one of the named measurements rather than
guessing.
3. **Act on finding [1] first.** Rules fire in priority order for a reason: a non-finite loss is
a numerics failure, not a modelling one, and a model that cannot overfit 20 examples has a
bug that no hyperparameter will fix.
4. **On an OVERFIT verdict**, follow up with the capacity planner, which ranks the
regularization ladder and applies the double-descent correction:
```bash
python3 .../capacity_planner.py --params <n> --train-examples <m> \
--train-error <x> --val-error <y> --applied early-stopping
```
5. **On a memory or throughput question**, run `model_arithmetic.py --spec <file>` — it reports
parameters, FLOPs and activation memory per example, and refuses a stack whose shapes do not
connect.
6. **Close with the discipline, not just the fix:** change one thing per experiment, log it,
re-measure the gap.
## Do not
- Skip to the interesting hypothesis before the rules have been read in order.
- Recommend collecting data while training error is high.
- Recommend shrinking an overparameterized model first — see the double-descent caveat in
`skills/deep-learning-book/references/book_to_2026_delta.md`.

39
engineering/deep-learning-book/commands/cs-dl-reading-path.md Normal file
View file

@ -0,0 +1,39 @@
---
name: "cs-dl-reading-path"
description: "/cs:dl-reading-path — Build a prerequisite-closed reading path through the Deep Learning book from a goal, a background and the hours you actually have. Refuses to route a goal the 2016 book does not cover, and names what covers it instead."
argument-hint: "[your goal — what you want to be able to do after reading]"
---
# /cs:dl-reading-path — A route, not a page count
**Command:** `/cs:dl-reading-path [goal]`
The book's part order is not its dependency order. Front-to-back means weeks in Chapters 2–4
before touching a network, which is where most readers stop.
## Procedure
1. **Ask three things** if the user has not said them, one at a time:
- What do you want to be able to *do* afterwards?
- Background: `none` / `math` / `applied` / `research`?
- Realistic study hours per week?
2. **Run the planner:**
```bash
python3 engineering/deep-learning-book/skills/deep-learning-book/scripts/reading_path_planner.py \
--goal "<goal>" --background <bg> --hours-per-week <n>
```
3. **Handle the refusals rather than working around them.**
- Exit 3 — the goal is outside the book. Relay what covers it; do not invent a path.
- Exit 4 — the goal is unroutable. Ask the printed questions, one per turn.
4. **Relay the path** with the hour budget, and flag that the hours are a planning heuristic to
recalibrate after chapter one.
5. **Offer the study scaffolding**: `skills/deep-learning-book/assets/study_log_template.md` and
`skills/deep-learning-book/assets/chapter_worksheet.md`, plus the retrieval-practice cadence from
`skills/deep-learning-book/references/study_method_canon.md`.
## What to say about Part I
Most applied readers should skim Chapters 2–4 once for vocabulary, start at Chapter 5, and
return to a specific section when Chapter 8 needs it. Readers heading for Part III should work
Part I properly — those chapters compound, and so do the gaps.
`skills/deep-learning-book/references/prerequisite_map.md` has the dependency graph and the three strategies.

194
engineering/deep-learning-book/skills/deep-learning-book/SKILL.md Normal file
View file

@ -0,0 +1,194 @@
---
name: deep-learning-book
description: "Study companion and working knowledge base for the Deep Learning textbook by Goodfellow, Bengio & Courville (MIT Press, 2016), read free at deeplearningbook.org. Indexes all 20 chapters, carries a 2016-to-2026 delta layer naming what the book got right, what was superseded (transformers, AdamW, diffusion, double descent) and what still holds, and ships four deterministic tools: a prerequisite-aware reading-path planner, a training-failure diagnostic, a capacity-and-regularization planner, and a parameter/FLOP/activation-memory calculator. Use when studying or teaching this book, planning a route through it, deciding whether a chapter's advice is still current, or translating its math into a training decision. It points at the official chapters — it never reproduces them."
license: MIT
metadata:
version: 1.0.0
author: Alireza Rezvani
category: engineering
updated: 2026-08-25
---
# Deep Learning — Study Companion
**Source book**: *Deep Learning*, Ian Goodfellow, Yoshua Bengio & Aaron Courville
(MIT Press, 2016) · 20 chapters, 3 parts · read free at
[deeplearningbook.org](https://www.deeplearningbook.org/) · companion compiled 2026-08-25.
**This is a companion, not a copy.** The book is copyrighted, and its site states that the
HTML-only format exists to discourage copying under the authors' MIT Press contract. Nothing
here reproduces its text. Every chapter file is original synthesis — what the chapter
establishes, how to use it, where it has aged — plus a link to the official chapter. Read the
book at the link; use this to navigate it, keep it current, and turn it into decisions.
See [references/rights_and_use.md](references/rights_and_use.md).
## How to Use This Skill
- **No argument** — load the core frameworks below.
- **A topic** — ask about `regularization`, `saddle points`, `partition function`; resolved
through the Topic Index, then that chapter file is read before answering.
- **`chNN`** — load that chapter's file.
- **"is this still true?"** — the 2016→2026 delta layer, in every chapter file and in
[references/book_to_2026_delta.md](references/book_to_2026_delta.md).
- **"where do I start?"** — run `scripts/reading_path_planner.py`.
When asked about something outside these 20 chapters, say so and route to the delta reference
rather than improvising the book's position on material published after it.
---
## Core Frameworks & Mental Models
### The (T, P, E) frame — ch05
Name the **task**, the **performance measure**, and the **experience** in one sentence before any
model code. Most failed projects failed at P: an unstated metric, or a proxy whose relationship
to the real objective was never checked.
### Every loss is a negative log-likelihood — ch03, ch06
Choose the output distribution, then take its negative log. Gaussian → MSE, Bernoulli → binary
cross-entropy, categorical → cross-entropy, Laplace → MAE. "Which loss?" is always the question
"which distribution?" in disguise. Modern contrastive and preference objectives sit outside this
frame — a real limit of the book, not a gap in your understanding.
### KL asymmetry decides your failure mode — ch03, ch19, ch20
D(p‖q) ≠ D(q‖p). Forward KL is mode-covering (blurry averages); reverse KL is mode-seeking
(sharp but partial). This single fact predicts VAE blur, GAN mode collapse, and the
characteristic over-confidence of mean-field variational posteriors.
### Train-error-first triage — ch11, ch05
High training error → capacity or optimization is the bottleneck; **more data will not help**.
Low training error with a large validation gap → data or regularization. This is the highest-value
heuristic in the book. `scripts/training_diagnostics.py` runs it.
### Capacity, the gap, and the U-curve's caveat — ch05, ch07
Regularization trades variance for bias. But the classical U-shaped capacity curve is incomplete:
past the interpolation threshold, test error can fall again (double descent, 2019–2020,
post-dating the book). Practical consequence: when a large model overfits, try more data, more
regularization or longer training **before** shrinking it.
### Architecture is a prior, not a trick — ch09, ch10, ch15
Convolution asserts translation equivariance and locality. Recurrence asserts that the past
compresses into a state. A distributed representation asserts that factors combine
combinatorially. When the assertion is false, the architecture cannot be rescued by tuning — and
when it is true, it beats capacity. This is also why Vision Transformers need more data than
ConvNets: they discard the prior and buy it back with examples.
### Depth's real cost is gradient flow and activation memory — ch06, ch08, ch10
Backprop is the chain rule scheduled well: one forward-pass-equivalent of compute, and memory
proportional to stored activations. Depth fails through vanishing/exploding gradients and
ill-conditioning, which is why residual connections, normalization and clipping exist.
### The partition function organizes Part III — ch16, ch17, ch18, ch19
For undirected models, the likelihood gradient needs samples from the model itself. Four escape
routes: sample it (CD/PCD), sidestep it algebraically (pseudolikelihood, **score matching**),
learn around it (NCE), or estimate it for evaluation (AIS). Score matching's descendants are
today's diffusion models — which is why Part III repays reading even though its models did not
survive.
### Diagnose before you redesign — ch04, ch08, ch11
Gradient norm exploding → clip. Norm large but loss flat → ill-conditioning. Norm near zero with
high loss → saturation or dead units. NaN → numerics first. Change one thing per experiment.
---
## Chapter Index
| # | Title | Key content |
|---|-------|-------------|
| [ch01](chapters/ch01-introduction.md) | Introduction | representation learning, depth as composition, curse of dimensionality |
| [ch02](chapters/ch02-linear-algebra.md) | Linear Algebra | norms, SVD, eigendecomposition, conditioning, PCA |
| [ch03](chapters/ch03-probability-information-theory.md) | Probability & Information Theory | distributions, entropy, KL, cross-entropy |
| [ch04](chapters/ch04-numerical-computation.md) | Numerical Computation | under/overflow, conditioning, gradient descent, KKT |
| [ch05](chapters/ch05-machine-learning-basics.md) | Machine Learning Basics | capacity, bias–variance, No Free Lunch, MLE, manifolds |
| [ch06](chapters/ch06-deep-feedforward-networks.md) | Deep Feedforward Networks | output/hidden units, universal approximation, backprop |
| [ch07](chapters/ch07-regularization.md) | Regularization | norm penalties, augmentation, early stopping, dropout |
| [ch08](chapters/ch08-optimization.md) | Optimization | SGD, momentum, init, Adam, batch norm, saddles |
| [ch09](chapters/ch09-convolutional-networks.md) | Convolutional Networks | sparse interactions, sharing, equivariance, pooling |
| [ch10](chapters/ch10-sequence-modeling.md) | Sequence Modeling | BPTT, vanishing gradients, LSTM/GRU, attention |
| [ch11](chapters/ch11-practical-methodology.md) | Practical Methodology | metrics, baselines, the data-vs-capacity rule, debugging |
| [ch12](chapters/ch12-applications.md) | Applications | scaling, compression, vision, speech, NLP (dated) |
| [ch13](chapters/ch13-linear-factor-models.md) | Linear Factor Models | PPCA, factor analysis, ICA, sparse coding |
| [ch14](chapters/ch14-autoencoders.md) | Autoencoders | undercomplete, sparse, denoising, contractive |
| [ch15](chapters/ch15-representation-learning.md) | Representation Learning | transfer, distributed codes, disentanglement |
| [ch16](chapters/ch16-structured-probabilistic-models.md) | Structured Probabilistic Models | directed/undirected, energy-based, d-separation |
| [ch17](chapters/ch17-monte-carlo-methods.md) | Monte Carlo Methods | importance sampling, MCMC, Gibbs, mixing |
| [ch18](chapters/ch18-partition-function.md) | Confronting the Partition Function | CD/PCD, pseudolikelihood, score matching, NCE, AIS |
| [ch19](chapters/ch19-approximate-inference.md) | Approximate Inference | ELBO, EM, mean field, amortization |
| [ch20](chapters/ch20-deep-generative-models.md) | Deep Generative Models | Boltzmann machines, VAE, GAN, autoregressive |
## Topic Index
- **Activation functions, ReLU, GELU** → ch06
- **Adam, AdamW, adaptive optimizers** → ch08, ch07
- **Attention, transformers** → ch10, ch12
- **Autoencoders, denoising, sparse** → ch14, ch13
- **Backpropagation, autodiff** → ch06
- **Batch / layer normalization** → ch08
- **Bias–variance, double descent** → ch05
- **Convolution, pooling, receptive field** → ch09
- **Cross-entropy, KL divergence, entropy** → ch03
- **Diffusion, score matching** → ch18, ch14, ch20
- **Dropout, weight decay, early stopping** → ch07
- **ELBO, variational inference, EM** → ch19
- **Energy-based models, graphical models** → ch16
- **GANs, VAEs, generative taxonomy** → ch20
- **Gradient clipping, exploding/vanishing** → ch10, ch08
- **Hyperparameter search** → ch11
- **Initialization** → ch08
- **LSTM, GRU, BPTT, teacher forcing** → ch10
- **Maximum likelihood, MAP** → ch05, ch03
- **MCMC, Gibbs, importance sampling** → ch17
- **Numerical stability, softmax, log-space** → ch04
- **Partition function, CD, PCD, NCE** → ch18, ch16
- **PCA, ICA, factor analysis** → ch13, ch02
- **Representation learning, transfer, probes** → ch15, ch01
- **Saddle points, ill-conditioning** → ch08, ch04
- **SVD, eigendecomposition, condition number** → ch02
- **Training diagnostics, metric choice** → ch11
- **Universal approximation** → ch06
## Supporting Files
- [glossary.md](glossary.md) — every key term with its chapter
- [patterns.md](patterns.md) — techniques as instruments, with trade-offs
- [cheatsheet.md](cheatsheet.md) — decision tables and defaults
- [references/book_to_2026_delta.md](references/book_to_2026_delta.md) — what changed, per chapter
- [references/prerequisite_map.md](references/prerequisite_map.md) — the real dependency graph
- [references/study_method_canon.md](references/study_method_canon.md) — how to study a hard text
- [references/rights_and_use.md](references/rights_and_use.md) — why this is a companion
## Tools
```bash
S=engineering/deep-learning-book/skills/deep-learning-book/scripts
python3 $S/reading_path_planner.py --goal "train a transformer" --background applied --hours-per-week 5
python3 $S/training_diagnostics.py --train-loss 0.02 --val-loss 1.9 --grad-norm 0.4 --epochs 30
python3 $S/capacity_planner.py --params 12000000 --train-examples 50000 --train-error 0.01 --val-error 0.22
python3 $S/model_arithmetic.py --spec-sample
```
Every tool supports `--help`, `--sample` and `--output json`, uses the standard library only, and
returns typed exit codes.
---
## Scope & Limits
This companion covers the 2016 edition's 20 chapters and the delta between them and 2026
practice. It does **not** cover: reinforcement learning beyond passing mention, LLM training
infrastructure, RLHF/DPO alignment, agentic systems, MLOps tooling, or fairness and safety
evaluation — none of which the book treats. For production ML engineering use
`engineering-team/senior-ml-engineer`; for LLM cost work use `engineering/llm-cost-optimizer`.
When a question lands outside the book, say the book does not cover it and cite the delta
reference for what replaced its position. A companion that quietly extrapolates is worse than one
that names its boundary.

62
engineering/deep-learning-book/skills/deep-learning-book/assets/chapter_worksheet.md Normal file
View file

@ -0,0 +1,62 @@
# Chapter Worksheet
One page per chapter. Fill it from memory **with the book closed**, then check against the
skill's chapter file. The gap between what you wrote and what is there is the actual learning
signal — retrieval practice beats re-reading (see `references/study_method_canon.md`).
---
**Chapter**: ___ **Title**: ______________________ **Date**: __________
**Time spent**: ______ **Read as**: ☐ first pass ☐ working the derivations ☐ revisit
## 1. Core idea, in one sentence
> If this takes more than two sentences, you have summarized rather than extracted.
_______________________________________________________________________________
## 2. Three concepts I could define to someone else
1. _____________________________________________________________________________
2. _____________________________________________________________________________
3. _____________________________________________________________________________
## 3. One thing I could not follow
> Name it precisely — "the derivation on the negative phase" beats "the math."
_______________________________________________________________________________
**Is it blocking?** ☐ yes, a later chapter needs it (which: ____) ☐ no, note and move on
## 4. Self-explanation prompt
Pick one equation. **What breaks if this term is removed?**
_______________________________________________________________________________
## 5. When would I actually use this?
> A note that answers this gets read again. A note that restates the text does not.
_______________________________________________________________________________
## 6. Still true in 2026?
Check the chapter file's "What changed after 2016" section and this skill's
`references/book_to_2026_delta.md`.
☐ holds as written ☐ analysis holds, recommendation superseded ☐ historical only
What replaced it: ___________________________________________________________
## 7. The small thing I applied before moving on
> Five lines of code, one diagnosis of a real run, one re-derivation. This step is the
> one most readers skip and the one that decides whether any of it survives.
_______________________________________________________________________________
## 8. Revisit date
☐ +1 week: ________ ☐ +1 month: ________ (ch05, ch08, ch11 and ch18 are worth both)

41
engineering/deep-learning-book/skills/deep-learning-book/assets/example_layer_spec.json Normal file
View file

@ -0,0 +1,41 @@
{
"name": "transformer encoder block, 768-dim, 512 tokens",
"_comment": "Feed to model_arithmetic.py --spec. Shapes are per example, no batch dim. The two feedforward linears run position-wise over the (512, 768) sequence: one weight matrix shared across all 512 tokens, which is what a transformer FFN actually is. Do NOT insert a flatten before them \u2014 that models a dense layer over the whole flattened sequence and inflates the parameter count by 512x. Note how the mha row's cost grows quadratically with sequence length: double 512 to 1024 and the attention term quadruples while the projections only double.",
"layers": [
{
"type": "input",
"shape": [
512,
768
],
"name": "token embeddings in"
},
{
"type": "layernorm",
"name": "pre-norm 1"
},
{
"type": "mha",
"heads": 12,
"name": "self-attention"
},
{
"type": "layernorm",
"name": "pre-norm 2"
},
{
"type": "linear",
"units": 3072,
"name": "ffn up (position-wise)"
},
{
"type": "activation",
"name": "gelu"
},
{
"type": "linear",
"units": 768,
"name": "ffn down (position-wise)"
}
]
}

51
engineering/deep-learning-book/skills/deep-learning-book/assets/study_log_template.md Normal file
View file

@ -0,0 +1,51 @@
# Study Log — Deep Learning (Goodfellow, Bengio & Courville)
Started: __________ · Goal: ______________________________________________
Path from `reading_path_planner.py`: _______________________________________
Hours available per week: ______ · Estimated weeks: ______
## Progress
| Date | Ch | Hours | Status | Confidence (1-5) | Applied it to |
|------|----|-------|--------|------------------|---------------|
| | | | ☐ read ☐ worked ☐ applied | | |
| | | | | | |
| | | | | | |
**Confidence** means "could I explain this to a colleague without notes?" — not "did I finish
the pages." A 2 is useful information; record it honestly and put the chapter on the revisit
list.
## Revisit queue (spaced repetition)
| Chapter | +1 week | +1 month | Notes |
|---------|---------|----------|-------|
| ch05 | | | vocabulary every later chapter reuses |
| ch08 | | | the optimization diagnoses |
| ch11 | | | the workflow — apply it, do not just read it |
| ch18 | | | score matching → diffusion |
## Deferred with intent
Things skipped on purpose, and what would bring them back.
| Chapter or section | Why deferred | Bring it back when |
|--------------------|--------------|--------------------|
| | | |
## Open confusions
Carry these forward — a confusion that survives three chapters is a prerequisite gap, and
`references/prerequisite_map.md` will name the chapter that fills it.
| # | Confusion | First hit (ch) | Resolved? |
|---|-----------|----------------|-----------|
| 1 | | | |
## Applied log
The point of the book. One line per real use.
| Date | What I was doing | Which chapter helped | Outcome |
|------|------------------|----------------------|---------|
| | | | |

59
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch01-introduction.md Normal file
View file

@ -0,0 +1,59 @@
# Chapter 1: Introduction
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/intro.html
## Core Idea
Deep learning is representation learning stacked: instead of hand-designing the features a
model consumes, you let the model learn a hierarchy of representations, each layer expressed
in terms of the one below it. Depth is the mechanism that makes composition cheap.
## Key Concepts
- **Representation learning**: learning the features, not just the mapping from features to output.
- **Depth as composition**: a deep model reuses sub-computations, so a function needing
exponentially many units at depth 1 may need polynomially many at depth k.
- **The curse of dimensionality**: as input dimension grows, the number of distinguishable
configurations grows exponentially, so local-interpolation methods run out of neighbours.
- **Prior of compositionality**: deep learning bets that the data was generated by a composition
of simple factors. This is a prior, not a theorem — where the bet is wrong, depth does not help.
- **AI winters and rebrands**: cybernetics → connectionism → deep learning; the ideas outlived
three funding cycles because the constraint was data and compute, not concept.
## Mental Models
- Think of each layer as **re-coordinatizing** the input so the next layer's job gets easier —
not as "extracting meaning."
- Prefer depth over width when the target function plausibly **factors**; prefer width when it
does not. Depth is a structural prior, and priors have to be earned.
- Treat "the model learns features automatically" as **shifting where the human judgment goes**
(architecture, objective, data curation), not eliminating it.
## Anti-patterns
- **Reading this chapter as history and skipping it**: the three growth curves it draws
(dataset size, model size, benchmark accuracy) are the argument for why anything else works.
- **Assuming depth is always better**: without the compositional structure, extra depth buys
optimization difficulty and nothing else.
## What changed after 2016
The chapter's central claim — capability tracks dataset and model scale — held, and was made
quantitative by neural scaling laws (Hestness et al. 2017; Kaplan et al. 2020; Hoffmann et al.
2022 "Chinchilla"), which the 2016 text predates. Its examples of "what deep learning can do"
are now badly dated; its explanation of *why* depth helps is not. **Confidence: high** — the
scaling-law literature is public and replicated.
## Key Takeaways
1. Before adding depth, state the compositional structure you believe the data has.
2. Use the curse of dimensionality as the test for whether a nearest-neighbour-style baseline
is even viable on your input.
3. Read the growth curves as a resource argument: most 2016-era "deep learning doesn't work
here" results were data or compute verdicts, not architecture verdicts.
## Connects To
- **Ch 5**: the formal learning-theory machinery behind the claims made informally here.
- **Ch 15**: representation learning, promised here and delivered there.
- **references/book_to_2026_delta.md**: the scaling-law layer this chapter now needs.

61
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch02-linear-algebra.md Normal file
View file

@ -0,0 +1,61 @@
# Chapter 2: Linear Algebra
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/linear_algebra.html
## Core Idea
The minimum linear algebra a deep learning practitioner needs, chosen for what appears later:
tensors and broadcasting, norms, eigendecomposition, SVD, the pseudoinverse, and PCA derived
from scratch. This is a filter, not a course — it teaches what Chapters 4, 5, 8 and 13 will use.
## Key Concepts
- **Tensor / broadcasting**: an n-dimensional array plus the rules for combining shapes. Most
real "model bugs" are shape bugs.
- **Norms**: L2 (Euclidean), L1 (sparsity-friendly, non-differentiable at 0), max-norm,
Frobenius. The choice of norm *is* the choice of what "small" means in a regularizer.
- **Eigendecomposition**: A = V diag(λ) V⁻¹ for square matrices; the eigenvalues tell you the
local curvature story later in Ch 4 and 8.
- **Singular value decomposition (SVD)**: A = U D Vᵀ, defined for *any* matrix. The general tool
where eigendecomposition needs square and diagonalizable.
- **Moore–Penrose pseudoinverse**: the least-squares / minimum-norm solution when a system is
over- or under-determined.
- **Condition number**: ratio of largest to smallest singular value; large means small input
perturbations produce large output changes — the numerical fragility Ch 4 confronts.
- **PCA**: derived here as the linear encoder/decoder minimizing L2 reconstruction error, which
is exactly the framing Ch 13–14 generalize.
## Mental Models
- Read a matrix as a **function on space**, and its singular values as how much it stretches
each orthogonal direction. Condition number = worst stretch / least stretch.
- Use SVD as the **default** decomposition and reach for eigendecomposition only when symmetry
buys you something (it does for Hessians).
- Treat a regularizer's norm as a **prior over parameter space**: L2 says "small and spread",
L1 says "mostly zero."
## Anti-patterns
- **Skipping to Chapter 6.** Ch 4 and 8's discussion of ill-conditioning is unreadable without
eigenvalues and condition number, and readers who skip typically bounce off Ch 8.
- **Memorizing decompositions as identities** instead of as geometry — the geometry is what
transfers to optimization.
## What changed after 2016
Nothing in the mathematics. What changed is the practice: in half-precision training, condition
number stopped being a theoretical concern and became an operational one (loss scaling, bf16 over
fp16 precisely because of dynamic range). **Confidence: high.**
## Key Takeaways
1. Track shapes explicitly; treat a shape mismatch as a modelling error, not a typing error.
2. Pick your norm deliberately when you regularize — you are choosing the shape of the prior.
3. Learn PCA in this chapter's form (encoder/decoder minimizing reconstruction) so Ch 13–14 read
as generalizations rather than new material.
## Connects To
- **Ch 4**: conditioning and numerical stability use exactly these quantities.
- **Ch 13**: PCA reappears as a linear factor model with an explicit probabilistic story.
- **Ch 8**: Hessian eigenvalues explain why gradient descent stalls.

67
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch03-probability-information-theory.md Normal file
View file

@ -0,0 +1,67 @@
# Chapter 3: Probability and Information Theory
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/prob.html
## Core Idea
Deep learning models are probability distributions, and losses are log-likelihoods. This chapter
installs the vocabulary — random variables, the common distributions, expectation, Bayes' rule,
entropy, KL divergence, cross-entropy — that makes "why this loss?" answerable rather than
conventional.
## Key Concepts
- **Frequentist vs Bayesian probability**: rates of events vs degrees of belief. Deep learning
uses the Bayesian reading for parameters and the frequentist one for evaluation, mostly
without saying so.
- **Marginal / conditional / chain rule**: the algebra every graphical model in Ch 16 runs on.
- **Common distributions**: Bernoulli, categorical, Gaussian, exponential, Laplace, Dirac,
empirical, and mixtures. Each corresponds to an output layer you will actually build.
- **Self-information and Shannon entropy**: surprise, and expected surprise.
- **KL divergence**: asymmetric. D(p‖q) ≠ D(q‖p), and the asymmetry decides whether your fitted
model covers all modes or concentrates on one.
- **Cross-entropy**: H(p,q) = H(p) + D(p‖q). Minimizing cross-entropy over q is minimizing KL,
because H(p) is constant in q — this is why classification uses it.
- **Structured probabilistic models**: factorization of a joint into conditionals over a graph.
## Mental Models
- Read every loss as a **negative log-likelihood** under an assumed output distribution: MSE is
a Gaussian with fixed variance, cross-entropy is a categorical, MAE is a Laplace. If you know
the assumed distribution, you know when the loss is wrong for your data.
- Use KL **direction** as a design lever: forward KL (data ‖ model) is mode-covering and gives
blurry averages; reverse KL (model ‖ data) is mode-seeking and gives sharp but partial fits.
This one fact explains most of Ch 19 and 20.
- Treat softmax as **exp-then-normalize on logits**, and remember from Ch 4 that it must be
computed in a shift-stabilized form.
## Anti-patterns
- **Choosing MSE for a bounded or count-valued target** — you have assumed a Gaussian on data
that is not Gaussian, and the residual structure will tell you so.
- **Reading KL as a distance**: it is not symmetric and does not satisfy the triangle inequality.
- **Interpreting softmax outputs as calibrated probabilities** without checking calibration —
the chapter's math does not promise calibration, and modern networks are typically
overconfident (Guo et al. 2017, post-dating the book).
## What changed after 2016
The probabilistic core is unchanged. Two additions matter: the calibration literature (deep
networks are systematically overconfident; temperature scaling is the cheap fix), and the rise
of losses that are *not* clean log-likelihoods — contrastive/InfoNCE objectives, and preference
losses such as DPO. The chapter's "every loss is a likelihood" framing needs that caveat now.
**Confidence: high** for calibration; **high** for the contrastive family being outside the
chapter's scope.
## Key Takeaways
1. State the output distribution before choosing the loss; the loss follows from it.
2. When a generative model looks blurry, suspect forward KL; when it looks mode-collapsed,
suspect reverse KL.
3. Check calibration separately from accuracy — the book does not, and the gap is real.
## Connects To
- **Ch 5**: maximum likelihood as the estimator that justifies these losses.
- **Ch 16–19**: graphical models, sampling, and variational inference all run on this algebra.
- **Ch 20**: the KL asymmetry decides GAN vs VAE failure modes.

66
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch04-numerical-computation.md Normal file
View file

@ -0,0 +1,66 @@
# Chapter 4: Numerical Computation
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/numerical.html
## Core Idea
Real arithmetic on finite hardware fails in specific, predictable ways — underflow, overflow,
ill-conditioning — and gradient-based optimization inherits every one of them. This is the
shortest chapter with the highest debugging payoff.
## Key Concepts
- **Underflow / overflow**: numbers rounding to zero (then dividing) or exceeding representable
range (then becoming inf/NaN).
- **Stabilized softmax**: subtract the max logit before exponentiating; mathematically identical,
numerically survivable. `log_softmax` exists for the same reason.
- **Conditioning**: how much a function's output moves for small input moves; for a matrix, the
condition number from Ch 2.
- **Gradient descent** and the first-order Taylor picture; **critical points**: minima, maxima,
saddles.
- **Jacobian and Hessian**; second-order Taylor expansion, and the optimal step size implied by
curvature.
- **Newton's method** and why it is not the default in deep learning: the Hessian is n×n in the
parameter count.
- **Constrained optimization / KKT**: the framing that makes Ch 7's norm penalties readable as
constraints.
## Mental Models
- Read a **NaN in the loss** as a numerics report first and a modelling bug second: check
log(0), division by a near-zero denominator, exp of a large logit, and an exploding gradient
in that order.
- Think of the Hessian's **eigenvalue spread as the terrain**: a large condition number is a
narrow ravine, and the largest safe step size is set by the largest eigenvalue while progress
is set by the smallest.
- Prefer the **log-domain** whenever probabilities are multiplied — this is the single highest
yield habit in this chapter.
## Anti-patterns
- **Hand-rolling softmax or cross-entropy** in a training loop instead of using the fused,
stabilized primitive.
- **Blaming the learning rate for every divergence**: ill-conditioning produces the same symptom
and does not respond to the same fix.
- **Reaching for a second-order optimizer** in a model with millions of parameters without
understanding the memory cost.
## What changed after 2016
Mixed-precision training (fp16 with loss scaling; then bf16, which trades mantissa bits for
exponent range specifically to avoid these failures) made this chapter's content a daily
operational concern rather than a background caution. Gradient clipping became standard practice
for transformer training. **Confidence: high.**
## Key Takeaways
1. Work in log-space by default for anything probabilistic.
2. When training diverges, separate "step too large" from "problem ill-conditioned" — they need
different fixes.
3. Understand KKT well enough to read weight decay as a constraint; Ch 7 assumes it.
## Connects To
- **Ch 8**: every optimization difficulty named here recurs there at scale.
- **Ch 7**: the constrained-optimization view of regularization.
- **scripts/training_diagnostics.py**: mechanizes the NaN/divergence triage above.

66
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch05-machine-learning-basics.md Normal file
View file

@ -0,0 +1,66 @@
# Chapter 5: Machine Learning Basics
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/ml.html
## Core Idea
Everything after this chapter is a special case of it: a task, a performance measure, an
experience, a capacity choice, and the generalization gap that capacity choice produces. If you
read one chapter of Part I, read this one.
## Key Concepts
- **Task / performance measure / experience (T, P, E)**: the definition that forces you to name
the metric before the model.
- **Capacity**: the range of functions a learner can express. Representational capacity vs
*effective* capacity (what the optimizer actually reaches).
- **Underfitting / overfitting** and the classical **U-shaped** generalization-error curve.
- **Bias–variance decomposition**: expected error splits into bias², variance, and irreducible
noise. Regularization trades variance for bias.
- **No Free Lunch theorem**: averaged over *all* data-generating distributions, every algorithm
ties. Therefore all progress comes from priors matched to the distributions we actually face.
- **Regularization**: any change intended to reduce generalization error but not training error.
- **Maximum likelihood estimation** and its **MAP** counterpart; consistency and efficiency.
- **Hyperparameters and validation sets**: never tune on test.
- **Curse of dimensionality, local constancy, manifold hypothesis**: why nearest-neighbour-style
priors fail and why deep models assume data concentrates near a low-dimensional manifold.
## Mental Models
- Ask "**what is the prior?**" before "what is the model?" — No Free Lunch says the prior is
where all the leverage is.
- Read regularization as **moving mass in the bias–variance budget**, so that "add dropout" and
"get more data" are alternative purchases of the same thing.
- Treat the **train/val gap** as your primary instrument: gap small + error high = underfit
(capacity or optimization); gap large = overfit (regularization or data).
## Anti-patterns
- **Tuning against the test set** — including "just peeking once."
- **Comparing models on different splits**, or reporting a single seed for a small dataset.
- **Believing the U-curve unconditionally.** See below: this is the one place where the 2016
text is now known to be incomplete.
## What changed after 2016
**This is the chapter's one substantive correction.** The classical U-shaped capacity curve is
not the whole picture. In the heavily overparameterized regime, test error can fall again past
the interpolation threshold — "double descent" (Belkin et al. 2019; Nakkiran et al. 2020), which
post-dates the book. Modern large models routinely sit in a regime the 2016 framing predicts
should overfit catastrophically and does not. The bias–variance decomposition remains correct as
algebra; the *managerial advice* "reduce capacity when you overfit" is no longer the only right
move. **Confidence: high** — double descent is widely replicated. Practical consequence: try
"more data / more regularization / train longer" before "smaller model."
## Key Takeaways
1. Write down T, P, E in one sentence before writing any model code.
2. Use the train/val gap to choose your next action; do not guess.
3. Hold the U-curve loosely — check whether you are past the interpolation threshold before
shrinking a model that overfits.
## Connects To
- **Ch 7**: every regularizer named here, in depth.
- **Ch 11**: this chapter's diagnostics turned into a workflow.
- **scripts/capacity_planner.py**: mechanizes the gap-to-action rule, double descent included.

61
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch06-deep-feedforward-networks.md Normal file
View file

@ -0,0 +1,61 @@
# Chapter 6: Deep Feedforward Networks
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/mlp.html
## Core Idea
An MLP is a composed chain of parametric functions trained by gradient descent, where the choice
of output unit follows from the assumed output distribution and the choice of hidden unit follows
from gradient behaviour. Backpropagation is not learning; it is the chain rule scheduled well.
## Key Concepts
- **Universal approximation**: one sufficiently wide hidden layer can approximate any continuous
function on a compact set. It says nothing about learnability, sample efficiency, or the size
required — depth is how you get the same function with far fewer units.
- **Output units follow the distribution**: linear + MSE → Gaussian; sigmoid + binary
cross-entropy → Bernoulli; softmax + cross-entropy → categorical; mixture density → multimodal.
- **Hidden units**: ReLU as the sane default; leaky/parametric ReLU, ELU, GELU/Swish (post-2016)
for smoothness; sigmoid/tanh only where saturation is desired (gates).
- **Saturation**: sigmoid/tanh gradients vanish in the tails, which is why they are poor hidden
units and fine as gates.
- **Backpropagation**: reverse-mode automatic differentiation over the computation graph. Cost is
roughly one forward pass, memory is the stored activations.
- **Computation graph / autodiff**: the abstraction every framework implements. Forward mode is
cheap in inputs; reverse mode is cheap in outputs — losses are scalar, hence reverse.
## Mental Models
- Pick the output unit from the **likelihood** (Ch 3), then pick the loss as its negative log.
Every "which loss?" question reduces to "which distribution?"
- Read **ReLU's advantage as gradient preservation**, not nonlinearity per se: it is piecewise
linear, so the gradient through an active unit is exactly 1.
- Treat activation memory as the **real** cost of depth in training: the backward pass needs the
forward activations, which is why checkpointing trades compute for memory.
## Anti-patterns
- **Citing universal approximation to justify a shallow model** — the theorem allows an
exponentially wide layer, which is not an engineering plan.
- **Sigmoid hidden layers** in a deep stack.
- **Pairing a sigmoid output with MSE**: gradients vanish exactly where the model is most wrong.
Use cross-entropy.
## What changed after 2016
Smooth activations (GELU, SiLU/Swish) became the default in transformers; gated variants
(GLU, SwiGLU) are now standard in large language model feedforward blocks. Residual connections
(He et al. 2015) are mentioned here only in passing but became the structural default for every
deep stack. Reverse-mode autodiff is unchanged. **Confidence: high.**
## Key Takeaways
1. Derive the loss from the output distribution rather than picking it by habit.
2. Default to ReLU-family hidden units; reserve saturating units for gates.
3. Budget activation memory as a first-class constraint, not an implementation detail.
## Connects To
- **Ch 3**: the distributions that determine output units.
- **Ch 8**: why gradient preservation matters at depth.
- **scripts/model_arithmetic.py**: parameter, FLOP and activation-memory accounting for a stack.

66
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch07-regularization.md Normal file
View file

@ -0,0 +1,66 @@
# Chapter 7: Regularization for Deep Learning
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/regularization.html
## Core Idea
Regularization is anything that trades training error for generalization error, and this chapter
is the catalogue: norm penalties, data augmentation, noise, early stopping, parameter sharing,
sparsity, ensembling, dropout, and adversarial training — with their equivalences made explicit.
## Frameworks Introduced
- **L2 / weight decay**: shrinks parameters along low-curvature Hessian directions. Under a
quadratic approximation, it damps each eigen-direction by λᵢ/(λᵢ+α) — directions the loss does
not care about get pulled to zero.
- **L1**: yields genuine sparsity; equivalent to a Laplace prior on parameters.
- **Norm penalty as constraint**: via KKT (Ch 4), a penalty is a soft version of a norm ball.
Explicit projection (max-norm) is sometimes better behaved.
- **Dataset augmentation**: the most reliable regularizer when the invariance is real. Choose
transformations that preserve the label — and check that they do.
- **Noise injection**: on inputs (≈ L2 penalty for some models), on weights (encourages flat
minima), on labels (label smoothing).
- **Early stopping**: the cheapest regularizer; under a quadratic approximation it is
approximately equivalent to L2 with a strength set by the number of steps.
- **Parameter tying / sharing**: the strongest form — it removes parameters instead of penalizing
them. Convolution is parameter sharing (Ch 9).
- **Bagging and ensembles**: variance reduction by averaging independently trained models.
- **Dropout**: approximate ensembling over exponentially many sub-networks at the cost of one.
- **Adversarial training**: penalizing sensitivity to worst-case local perturbation.
## Mental Models
- Order regularizers by **cost per unit of gap closed**: more real data > augmentation > early
stopping > weight decay > dropout > architecture surgery.
- Read weight decay as **"forget the directions the data does not constrain"** — that is exactly
what the eigen-analysis says.
- Treat dropout as an **ensemble**, so its interaction with batch normalization (train/test
statistics mismatch) is expected rather than surprising.
## Anti-patterns
- **Stacking every regularizer at once** and then tuning — you cannot attribute the effect.
- **Augmenting with label-destroying transforms** (horizontal flip on digits, aggressive crops on
fine-grained classes).
- **Dropout inside a residual transformer block plus batch norm plus heavy weight decay** without
measuring; modern stacks regularize far more lightly than 2016 practice.
## What changed after 2016
Weight decay and L2 were shown to be *not* equivalent under adaptive optimizers, which is why
**AdamW** (Loshchilov & Hutter 2017/2019) decouples them — this post-dates the book and is now
the default. Dropout largely left large-scale vision and language models, displaced by
normalization, augmentation and sheer data volume. Label smoothing and stochastic depth became
common. **Confidence: high** for AdamW; **high** for the decline of dropout at scale.
## Key Takeaways
1. Add regularizers one at a time and measure the gap after each.
2. Use decoupled weight decay (AdamW) rather than L2-in-the-loss with an adaptive optimizer.
3. Prefer parameter sharing over penalties when a real invariance exists.
## Connects To
- **Ch 5**: the bias–variance budget these all spend from.
- **Ch 8**: optimizer choice changes what weight decay means.
- **Ch 11**: which regularizer to reach for, given a measured gap.

67
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch08-optimization.md Normal file
View file

@ -0,0 +1,67 @@
# Chapter 8: Optimization for Training Deep Models
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/optimization.html
## Core Idea
Training is not pure optimization: you minimize an empirical surrogate to reduce a risk you
cannot measure, on a non-convex landscape, with noisy gradients. The chapter separates the
difficulties (ill-conditioning, saddles, cliffs, long-term dependencies, poor correspondence
between local and global structure) from the algorithms that address them.
## Frameworks Introduced
- **Empirical risk minimization and its surrogates**: you optimize a differentiable proxy, not
the metric you report.
- **Minibatch SGD**: gradient noise scales roughly with 1/√batch, so bigger batches buy accuracy
in the gradient at linear cost — a poor trade past a point.
- **Ill-conditioning**: the dominant obstacle in practice; gradient norm can *grow* while the loss
stalls.
- **Saddle points, not local minima**: in high dimensions, critical points are overwhelmingly
saddles. This reframes the folklore fear of local minima.
- **Cliffs and exploding gradients** → **gradient clipping**.
- **Momentum** and **Nesterov momentum**: accumulate a velocity to cross ravines.
- **Initialization**: Xavier/Glorot and He scaling to keep activation and gradient variance
roughly constant with depth. Initialization is an algorithm, not a detail.
- **Adaptive methods**: AdaGrad (decaying), RMSProp, **Adam** (RMSProp + momentum + bias
correction).
- **Second-order methods**: Newton, conjugate gradient, BFGS/L-BFGS — and why they rarely survive
contact with minibatch noise and parameter counts.
- **Batch normalization**: reparameterization that stabilizes the scale of layer inputs.
- **Curriculum learning / coordinate descent / Polyak averaging**: the meta-strategies.
## Mental Models
- Diagnose by **what the gradient is doing**: norm exploding = cliff, clip. Norm large but loss
flat = ill-conditioning, use momentum/adaptive/normalization. Norm ~0 with high loss = dead
units or saturation.
- Treat **learning-rate schedule as a first-class hyperparameter** — usually more important than
the choice among SGD/Adam variants.
- Read batch norm as **making the loss surface better conditioned**; whether it does so by
reducing "internal covariate shift" is contested (Santurkar et al. 2018, post-2016).
## Anti-patterns
- **Fearing local minima**: spend the debugging effort on conditioning and learning rate.
- **Tuning the optimizer before the learning rate.**
- **Batch norm with tiny batches** — the batch statistics become noise. Use group/layer norm.
## What changed after 2016
The largest delta in Part II. **AdamW** replaced Adam+L2. Learning-rate **warmup plus cosine
decay** became standard for transformers. **LayerNorm** (and RMSNorm) displaced batch norm in
sequence models; pre-norm residual placement displaced post-norm for deep stacks. Large-batch
training with LARS/LAMB, and the linear-scaling-rule literature, arrived after the book.
**Confidence: high.**
## Key Takeaways
1. Tune learning rate and schedule first; optimizer family second.
2. Read gradient-norm behaviour before changing the architecture.
3. Use He/Xavier-style initialization deliberately — bad init is a common silent failure.
## Connects To
- **Ch 4**: conditioning, Hessians, and step size.
- **Ch 10**: exploding/vanishing gradients as the sequence-model version of these problems.
- **scripts/training_diagnostics.py**: this chapter's triage as an executable decision tree.

66
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch09-convolutional-networks.md Normal file
View file

@ -0,0 +1,66 @@
# Chapter 9: Convolutional Networks
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/convnets.html
## Core Idea
Convolution is three ideas at once — sparse interactions, parameter sharing, and equivariance to
translation — and each is a prior about grid-structured data. Pooling adds approximate invariance
to small translations. The architecture is a statement about the data, not a trick.
## Frameworks Introduced
- **Sparse interactions**: each output depends on a small receptive field, so cost drops from
O(m·n) to O(k·n).
- **Parameter sharing**: one kernel is reused at every position — the strongest regularizer in
Ch 7's catalogue, applied structurally.
- **Equivariance to translation**: shift the input, the feature map shifts. Convolution is *not*
equivariant to rotation or scale — a fact that motivates augmentation.
- **Pooling**: max/average pooling gives local invariance and downsampling; it is a prior that
small position changes should not matter.
- **Variants**: valid/same/full padding, strided convolution, dilated convolution, tiled
convolution, locally-connected layers (no sharing), transposed convolution for upsampling.
- **Convolution as an infinitely strong prior**: the chapter's sharpest framing — a conv layer is
a fully-connected layer with hard constraints on its weights.
- **Structured outputs**: dense prediction (segmentation) rather than one label per image.
- **Efficiency**: FFT-based and separable convolutions.
## Mental Models
- Ask "**is the statistic I need position-invariant?**" If yes, convolve. If not (a face-aligned
dataset, a tabular grid with meaningful coordinates), the prior is wrong and locally-connected
or attention layers may fit better.
- Compute **receptive field** deliberately: it must cover the evidence needed for the decision.
Depth, stride and dilation are three ways to buy it, with different costs.
- Treat pooling as **throwing away location on purpose**; when location is the answer
(segmentation, detection), pool less and use dilation or skip connections.
## Anti-patterns
- **Using convolution on data with no spatial/temporal locality** (arbitrary tabular columns) —
the sharing prior is simply false there.
- **Ignoring receptive field** and then adding parameters to fix an underfitting model.
- **Aggressive pooling in a dense-prediction task.**
## What changed after 2016
Residual networks (2015) are treated only briefly here but became the default. Since then:
depthwise-separable convolutions (MobileNet/Xception), EfficientNet-style compound scaling,
and — most importantly — **Vision Transformers** (Dosovitskiy et al. 2020), which discard the
convolutional prior in favour of data plus attention, and win at scale while ConvNets remain
competitive at smaller data sizes (ConvNeXt, 2022, closed much of the gap). The chapter's
argument that the conv prior is a *bet on the data* is exactly what ViT's data-hunger confirms.
**Confidence: high.**
## Key Takeaways
1. Justify convolution by the invariance you actually believe in.
2. Size the receptive field before adding depth for its own sake.
3. Expect the conv prior to pay off most when data is limited — this is the modern boundary
between ConvNets and ViTs.
## Connects To
- **Ch 7**: parameter sharing as structural regularization.
- **Ch 12**: computer-vision applications built on this.
- **references/book_to_2026_delta.md**: the ViT/ConvNeXt line.

70
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch10-sequence-modeling.md Normal file
View file

@ -0,0 +1,70 @@
# Chapter 10: Sequence Modeling — Recurrent and Recursive Nets
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/rnn.html
## Core Idea
Recurrence shares parameters across time the way convolution shares them across space, which lets
one model handle variable-length sequences. The price is that gradients must traverse many
multiplicative steps, producing the vanishing/exploding gradient problem — and gated
architectures exist to pay it.
## Frameworks Introduced
- **Unfolding the computational graph**: a recurrent definition becomes a deep feedforward graph
with tied weights.
- **BPTT (backpropagation through time)** and **truncated BPTT**.
- **Teacher forcing**: train on ground-truth previous tokens; note the train/inference mismatch
(exposure bias) it creates.
- **Vanishing and exploding gradients**: repeated multiplication by the recurrent Jacobian; the
spectral radius decides which.
- **Gradient clipping** for the exploding half; **gating** for the vanishing half.
- **LSTM** (input/forget/output gates plus a cell with an additive path) and **GRU** (a
two-gate simplification). The additive cell path is the mechanism — it makes the gradient path
through time approximately linear.
- **Bidirectional RNNs**; **encoder–decoder / sequence-to-sequence** with a fixed-size context.
- **Attention** appears here as the fix for the encoder–decoder bottleneck.
- **Deep RNNs, recursive (tree-structured) nets, echo state networks, leaky units, skip
connections through time.**
## Mental Models
- Read gating as **learned, data-dependent memory management**: the forget gate decides how long
the additive highway stays open.
- Treat the fixed-size context vector in vanilla seq2seq as a **bottleneck** — the whole reason
attention was invented.
- Expect **exposure bias** whenever you teacher-force: the model has never seen its own mistakes
during training.
## Anti-patterns
- **Reaching for an RNN by default in 2026** for a task where a transformer is the standard
baseline and the sequence fits in context.
- **Training a long-sequence RNN without gradient clipping.**
- **Ignoring the train/inference mismatch** in autoregressive generation.
## What changed after 2016
**The largest single delta in the book.** Attention, described here as an enhancement to
recurrent seq2seq, became the whole architecture with *Attention Is All You Need* (Vaswani et al.
2017) — one year after publication. Transformers replaced RNNs for essentially all large-scale
sequence work: parallel training over sequence positions, direct O(1) path length between any two
tokens, and much better scaling. What survived: gradient clipping, teacher forcing, exposure bias,
and the vanishing/exploding analysis. What returned: linear-time recurrent architectures
(S4/Mamba-style state-space models, 2021–2024) as a long-context alternative, which makes this
chapter's material relevant again rather than obsolete. **Confidence: high.**
**Read this chapter for the gradient-flow analysis, not for the architecture recommendation.**
## Key Takeaways
1. Learn the vanishing/exploding analysis here — it explains residual connections, LayerNorm
placement, and state-space models alike.
2. Clip gradients whenever a recurrence is in the graph.
3. Treat this chapter's architecture advice as historical; treat its diagnosis as current.
## Connects To
- **Ch 8**: the same optimization pathologies, in the time dimension.
- **Ch 12**: NLP and speech applications built on these.
- **references/book_to_2026_delta.md**: the transformer displacement, in detail.

63
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch11-practical-methodology.md Normal file
View file

@ -0,0 +1,63 @@
# Chapter 11: Practical Methodology
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/guidelines.html
## Core Idea
Knowing many algorithms matters less than knowing which one to reach for given a measurement.
This chapter is a workflow: pick a metric, build an end-to-end baseline fast, then let
instrumentation — not intuition — decide whether to add data, capacity, or regularization.
## Frameworks Introduced
- **The design process**: (1) determine goals — the error metric and the target value; (2) build
a working end-to-end pipeline early; (3) instrument it to find bottlenecks; (4) change one
thing at a time based on measurement.
- **Choose the metric before the model**: accuracy, precision/recall, F-score, PR/ROC curves,
coverage. Name the target value and where it came from.
- **Sensible baselines**: pick the standard architecture and optimizer for the data type before
inventing anything.
- **More data or a bigger model?** — the decision procedure: if training error is high, the model
or the optimization is the bottleneck (data will not help). If training error is low and test
error is high, gather more data or regularize.
- **Hyperparameter tuning**: manual (understand what each knob does to effective capacity),
**grid search** (poor scaling in dimensions), **random search** (better — it does not waste
trials on unimportant dimensions; Bergstra & Bengio 2012), and model-based/Bayesian.
- **Debugging strategies**: visualize the model's actual predictions and worst cases; fit a tiny
subset to zero training error; compare backprop against numerical derivatives; monitor
activation and gradient histograms.
## Mental Models
- Use the **train-error-first rule** as the single most valuable heuristic in the book: high
training error means "do not collect data yet."
- Treat "**can it overfit 20 examples?**" as the smoke test that separates a bug from a modelling
limitation. If it cannot, you have a bug.
- Prefer **random search over grid search** whenever the number of hyperparameters exceeds ~2.
## Anti-patterns
- **Tuning many things per experiment** — you learn nothing attributable.
- **Collecting more data to fix underfitting.**
- **Optimizing a proxy metric** whose relationship to the real objective was never checked.
- **Skipping the end-to-end pipeline** in favour of perfecting one component.
## What changed after 2016
The workflow is the most durable material in the book and is now the backbone of MLOps practice.
Additions since: experiment tracking as standard tooling, seeded reproducibility expectations,
Hyperband/ASHA for early-stopping-based search (post-2016), and — for large models — scaling laws
used to *predict* the return on more data or parameters instead of testing empirically at full
size. **Confidence: high.**
## Key Takeaways
1. Write the metric and its target number before writing model code.
2. Ask "is training error high?" before every data or capacity decision.
3. Change one thing per experiment and log it.
## Connects To
- **Ch 5**: the underfit/overfit framing this operationalizes.
- **Ch 7 / Ch 8**: the two toolboxes the workflow selects from.
- **scripts/training_diagnostics.py**: this decision tree, executable.

62
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch12-applications.md Normal file
View file

@ -0,0 +1,62 @@
# Chapter 12: Applications
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/applications.html
## Core Idea
How deep learning was actually deployed circa 2016: large-scale implementation (GPUs,
distributed training, model compression), then computer vision, speech recognition, and NLP.
This is the most time-dated chapter in the book — read the *systems* half, discount the
*state-of-the-art* half.
## Frameworks Introduced
- **Large-scale implementation**: GPU data parallelism, model parallelism, asynchronous SGD
(Hogwild-style), parameter servers.
- **Model compression**: distillation, quantization, pruning — for inference cost.
- **Dynamic structure / conditional computation**: cascades and gating so that not every input
pays the full cost.
- **Preprocessing in vision**: contrast normalization, whitening, dataset augmentation.
- **Speech**: the acoustic-model pipeline and its shift from GMM-HMM to deep networks.
- **NLP**: n-gram models, word embeddings and the curse of dimensionality over vocabularies,
hierarchical softmax and sampling-based approximations for large output vocabularies,
neural machine translation.
- **Recommender systems** and the exploration/exploitation problem.
- **Knowledge representation and relational reasoning.**
## Mental Models
- Separate **capability claims** (perishable) from **systems constraints** (durable). Memory
bandwidth, batch efficiency, and inference cost still shape architecture choices.
- Read the large-vocabulary softmax section as an instance of the general pattern: **when the
normalizing sum is expensive, approximate it** — the same problem Ch 18 attacks head-on.
- Treat conditional computation as the ancestor of **Mixture-of-Experts** routing.
## Anti-patterns
- **Citing this chapter's benchmark numbers or SOTA claims** — they are a decade old.
- **Copying its NLP pipeline**: subword tokenization (BPE/SentencePiece) and pretrained
transformers replaced nearly all of it.
## What changed after 2016
Almost everything at the application layer. Speech moved to end-to-end CTC/attention models and
then to large self-supervised encoders. NLP moved to pretrained transformers (BERT 2018, GPT
family), with subword tokenization and full-softmax over ~30k–200k subwords making hierarchical
softmax largely unnecessary. Vision moved to self-supervised pretraining and ViTs. Distillation,
quantization and pruning grew into a mature inference-optimization discipline. Conditional
computation matured into sparse MoE. **Confidence: high.**
## Key Takeaways
1. Read this chapter for the systems reasoning and the approximation techniques, not for what is
state of the art.
2. When your output vocabulary or normalizing constant is huge, recognize it as the recurring
partition-function problem.
3. Plan inference cost as an architecture constraint from the start.
## Connects To
- **Ch 18**: the partition-function problem in its general form.
- **Ch 9 / Ch 10**: the architectures being applied here.
- **references/book_to_2026_delta.md**: what replaced each application pipeline.

60
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch13-linear-factor-models.md Normal file
View file

@ -0,0 +1,60 @@
# Chapter 13: Linear Factor Models
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/linear_factors.html
## Core Idea
The simplest generative models: sample latent factors from a simple prior, map them linearly to
observations, add noise. Probabilistic PCA, factor analysis, ICA, sparse coding and slow feature
analysis are all this one template with different priors — and they are the scaffolding for
everything in Chapters 14–20.
## Key Concepts
- **The template**: h ~ p(h); x = Wh + b + noise. Change p(h) and the noise model, get a
different named method.
- **Factor analysis**: Gaussian latent prior, diagonal (per-feature) observation noise.
- **Probabilistic PCA**: factor analysis with isotropic noise; recovers PCA as noise → 0.
- **Independent component analysis (ICA)**: non-Gaussian independent latents — this is what makes
the factors identifiable, which Gaussian models cannot be (any rotation fits equally well).
- **Sparse coding**: a heavy-tailed (Laplace/Cauchy) prior; inference is an optimization, not a
closed form, which makes encoding expensive.
- **Slow feature analysis**: a prior that useful factors change slowly over time.
- **Manifold interpretation of PCA**: the model concentrates probability near a linear subspace.
## Mental Models
- Read each model as **a prior choice**, and read the prior as the assumption that buys
identifiability. Gaussian latents are rotation-invariant, so a Gaussian model cannot tell you
*which* factors — only which subspace.
- Treat sparse coding as the moment where **inference becomes iterative**; that cost is precisely
what autoencoders (Ch 14) amortize with a learned encoder.
- Use these as the **linear baseline** for any representation-learning claim: if a linear factor
model matches your deep encoder, the depth is not earning its cost.
## Anti-patterns
- **Skipping this chapter and starting at Ch 20** — VAEs read as arbitrary machinery without the
latent-variable template established here.
- **Expecting interpretable factors from a Gaussian-latent model.**
## What changed after 2016
The template survived; the emphasis moved. Nonlinear ICA identifiability results (Hyvärinen et
al., 2016–2020) clarified when latent factors are recoverable at all, and the disentanglement
literature — notably Locatello et al. (2019) — showed that unsupervised disentanglement is
impossible without inductive biases or supervision, which is a formal statement of this
chapter's identifiability point. Sparse coding returned as a tool for interpreting neural
networks (sparse autoencoders over LLM activations, 2023–2024). **Confidence: high.**
## Key Takeaways
1. Name the latent prior when you propose any generative model; it determines what is learnable.
2. Use a linear factor model as the baseline before claiming a deep representation helps.
3. Remember that identifiability, not fit quality, is what non-Gaussian priors buy.
## Connects To
- **Ch 14**: autoencoders amortize the inference these models do by optimization.
- **Ch 15**: what makes a representation good.
- **Ch 20**: the VAE is this template with a nonlinear decoder and amortized inference.

64
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch14-autoencoders.md Normal file
View file

@ -0,0 +1,64 @@
# Chapter 14: Autoencoders
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/autoencoders.html
## Core Idea
An autoencoder learns to copy its input imperfectly, and the constraint that prevents perfect
copying is where the learning happens. Undercompleteness, sparsity, denoising and contraction are
four different ways to impose that constraint — each defines a different notion of "useful".
## Frameworks Introduced
- **Undercomplete autoencoder**: bottleneck smaller than the input. With linear units and MSE it
recovers PCA's subspace.
- **Regularized autoencoders**: capacity can exceed the input dimension as long as something else
prevents identity — this is the chapter's key move.
- **Sparse autoencoder**: penalize code activation; interpretable as a latent prior (Ch 13).
- **Denoising autoencoder (DAE)**: corrupt the input, reconstruct the clean version. The learned
map estimates the *score* — it points back toward the data manifold, which is the direct
ancestor of score-based diffusion models.
- **Contractive autoencoder (CAE)**: penalize the Jacobian norm of the encoder, so the
representation resists input perturbation except along the manifold.
- **Manifold learning view**: the encoder is sensitive along tangent directions of the data
manifold and insensitive orthogonal to it.
- **Stochastic encoders and decoders**: autoencoders as p(h|x) and p(x|h).
- **Predictive sparse decomposition**; **applications**: dimensionality reduction, semantic
hashing, pretraining.
## Mental Models
- Ask "**what stops it from learning the identity?**" — the answer names the inductive bias, and
a model with no answer learns nothing useful.
- Read a DAE's learned vector field as **pointing uphill in density**. Once you see that,
diffusion models are the same idea run at many noise levels.
- Treat the encoder as **amortized inference**: it replaces the per-example optimization of
sparse coding with one forward pass.
## Anti-patterns
- **An overcomplete autoencoder with no regularizer** — it can and will learn a copy.
- **Judging representation quality by reconstruction error**: low reconstruction error can mean
the code memorized the input, which is the opposite of useful.
## What changed after 2016
The denoising idea became the foundation of modern generative modelling: denoising score matching
(Vincent 2011) → score-based models (Song & Ermon 2019) → DDPM (Ho et al. 2020) → the diffusion
family. Greedy layerwise autoencoder pretraining, presented here as a live technique, disappeared
— displaced first by better initialization/normalization and then by large-scale supervised and
self-supervised pretraining. The VAE overshadowed the deterministic autoencoders for generation,
while masked autoencoders (He et al. 2021) revived the corruption idea for vision pretraining.
**Confidence: high.**
## Key Takeaways
1. Always name the constraint that prevents identity before training an autoencoder.
2. Do not use reconstruction error as a representation-quality metric; evaluate downstream.
3. Learn the denoising/score connection here — it is the cheapest on-ramp to diffusion models.
## Connects To
- **Ch 13**: sparse coding, amortized.
- **Ch 15**: what a "good" representation means.
- **Ch 20**: VAEs, and the diffusion line that grew from denoising.

64
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch15-representation-learning.md Normal file
View file

@ -0,0 +1,64 @@
# Chapter 15: Representation Learning
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/representation.html
## Core Idea
What makes one representation better than another, and how do you get one without labels? The
chapter names the properties worth wanting — distributed, disentangled, capturing causal factors
— and the transfer mechanisms that let a representation earned on one task pay off on another.
## Frameworks Introduced
- **Greedy layer-wise unsupervised pretraining**: the historically important technique that
restarted the field in 2006; the chapter is already candid that its value had shrunk by 2016.
- **Transfer learning and domain adaptation**: reuse a representation across tasks or
distributions; **one-shot / zero-shot learning** as the limiting cases.
- **Distributed representations**: n binary features describe 2ⁿ regions, versus n regions for a
one-hot/symbolic code. This exponential advantage is the central argument for learned features.
- **Disentangled factors**: separate underlying causes onto separate directions.
- **Exponential gains from depth**: deeper composition of distributed features again multiplies
expressible structure.
- **Causal factors and semi-supervised learning**: unsupervised learning helps supervised learning
exactly when p(x) and p(y|x) share structure — if the factors generating x include y's causes.
- **Regularization priors that define good representations**: smoothness, linearity, multiple
explanatory factors, hierarchy, sparsity, simplicity of factor dependencies, shared factors
across tasks, manifolds, temporal/spatial coherence.
## Mental Models
- Judge a representation by **what becomes linearly separable** in it — that is the operational
version of "good features."
- Use the **shared-cause test** to predict whether unlabeled data will help: if p(x) tells you
nothing about p(y|x), self-supervision will not rescue a supervised task.
- Read distributed-vs-symbolic as **the reason embeddings beat lookup tables**, and note that the
advantage is combinatorial, not merely empirical.
## Anti-patterns
- **Expecting unsupervised pretraining to help unconditionally** — the chapter itself is careful
here, and the 2016-era conclusion (it often does not, for large labeled datasets) was correct
for the methods then available.
- **Claiming disentanglement without a metric or an intervention.**
## What changed after 2016
This chapter aged into relevance rather than out of it. Self-supervised learning became the
dominant paradigm — contrastive methods (SimCLR, MoCo, CLIP), masked prediction (BERT, MAE), and
next-token prediction at scale — vindicating the chapter's core bet while replacing every
specific method it lists. Two corrections: greedy layer-wise pretraining is now purely
historical, and unsupervised disentanglement was proven impossible without inductive bias
(Locatello et al. 2019). Linear-probe evaluation became the standard test of representation
quality. **Confidence: high.**
## Key Takeaways
1. Evaluate representations with linear probes and downstream transfer, not reconstruction.
2. Before investing in self-supervision, argue that p(x) and p(y|x) share causes.
3. Treat "disentangled" as a claim requiring an intervention-based test.
## Connects To
- **Ch 14**: autoencoders as one route to a representation.
- **Ch 1**: the promise made in the introduction, cashed out here.
- **references/book_to_2026_delta.md**: the self-supervised learning line.

64
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch16-structured-probabilistic-models.md Normal file
View file

@ -0,0 +1,64 @@
# Chapter 16: Structured Probabilistic Models for Deep Learning
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/graphical_models.html
## Core Idea
A joint distribution over n variables is exponentially large; a graph that encodes conditional
independence makes it tractable. Directed models factor into conditionals, undirected models into
unnormalized potentials divided by a partition function — and that partition function is the bill
Chapters 17–19 spend their pages paying.
## Frameworks Introduced
- **The challenge of unstructured modelling**: memory, statistical efficiency, and inference cost
all scale with the number of parameters in a full joint.
- **Directed models (Bayesian networks)**: p(x) = ∏ p(xᵢ | parents(xᵢ)). Normalized by
construction — no partition function.
- **Undirected models (Markov random fields)**: p(x) = (1/Z) ∏ φ_c(x_c) over cliques. Z is the
partition function, and it is a sum over all configurations.
- **Energy-based models**: p(x) ∝ exp(−E(x)). Any positive distribution can be written this way.
- **Separation / d-separation**: reading conditional independence off the graph.
- **Converting between graph types; factor graphs** for disambiguating factorization.
- **Sampling from graphical models**: ancestral sampling (easy, directed) vs Gibbs sampling
(needed for undirected).
- **Structure learning and latent variables**; the **restricted Boltzmann machine (RBM)** as the
worked example.
## Mental Models
- Use the rule of thumb: **directed = easy sampling, harder inference with explaining-away;
undirected = natural for mutual constraints, hard normalization.**
- Read an energy function as a **soft constraint set**: low energy where constraints are
satisfied. Design E, and p follows.
- Remember that the partition function is not an inconvenience — it is **the** obstacle that
organizes all of Part III.
## Anti-patterns
- **Writing an undirected model and ignoring Z** until training refuses to work.
- **Assuming a graph's missing edge means independence in the data** — it means the *model*
asserts independence.
## What changed after 2016
Deep undirected graphical models (deep Boltzmann machines, RBM stacks) are now largely historical
as generative workhorses. But energy-based modelling did not die: it returned via score matching
and diffusion (which sidestep Z entirely by learning ∇ log p rather than p), and via
energy-based reinterpretations of contrastive learning. Autoregressive factorization — the
directed side of this chapter — became the dominant paradigm through transformer language models,
which are exactly chain-rule factorizations with a huge neural conditional. **Confidence: high.**
## Key Takeaways
1. Classify any generative proposal as directed or undirected first; it predicts which problems
you will have.
2. When Z is intractable, look for a formulation that never needs it (score, ratio, or
autoregressive).
3. Read modern LLMs as ancestral sampling from a directed chain — the framing here still applies.
## Connects To
- **Ch 17–19**: the three responses to intractability (sample it, approximate Z, approximate the
posterior).
- **Ch 20**: the models built on these foundations.

60
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch17-monte-carlo-methods.md Normal file
View file

@ -0,0 +1,60 @@
# Chapter 17: Monte Carlo Methods
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/monte_carlo.html
## Core Idea
When a sum or integral is intractable, estimate it with samples. The estimator is unbiased and
its error shrinks as 1/√n regardless of dimension — which is why sampling survives where
quadrature does not. The difficulty moves from computing the integral to obtaining the samples.
## Key Concepts
- **Monte Carlo estimation**: approximate E_p[f(x)] by the sample mean; variance falls as 1/n,
standard error as 1/√n.
- **Importance sampling**: sample from a tractable q and reweight by p/q. Variance depends
entirely on how well q matches p·f — a bad proposal gives an estimator that is unbiased and
useless.
- **Markov chain Monte Carlo (MCMC)**: build a chain whose stationary distribution is the target,
then use its states as (correlated) samples.
- **Gibbs sampling**: resample each variable from its conditional given the rest; the natural
MCMC scheme for graphical models.
- **Burn-in / mixing time**: the chain needs time to forget its initialization and to move
between modes.
- **The mixing problem between separated modes**: with well-separated modes, transition
probability between them is tiny and the chain reports a single mode as if it were everything.
- **Tempering / annealing** as remedies: flatten the distribution so the chain can travel.
## Mental Models
- Treat a Monte Carlo estimate as a **measurement with error bars**; report the standard error,
because "the estimate is unbiased" says nothing about whether n was large enough.
- Diagnose a suspiciously confident sampler as a **mixing failure**, not a modelling success.
- Read importance sampling's variance condition as: **you must already know roughly where the
mass is** — this is why proposals matter more than sample counts.
## Anti-patterns
- **Reporting MCMC samples without a mixing diagnostic.**
- **Trusting importance weights with huge dynamic range** — an effective sample size of ~1 is
common and invisible unless measured.
## What changed after 2016
The chapter's methods remain textbook-correct, but their role in deep generative modelling
shrank: diffusion models replaced slow MCMC-based sampling with a fixed, finite denoising chain,
and modern generation is dominated by ancestral sampling from autoregressive models. Where MCMC
is still used — Bayesian deep learning, some energy-based models — HMC/NUTS variants dominate
over plain Gibbs. **Confidence: high.**
## Key Takeaways
1. Always pair a Monte Carlo estimate with its standard error.
2. Check mixing before believing anything an MCMC sampler tells you about multimodality.
3. Recognize the pattern "intractable expectation → sample it" — it recurs throughout Ch 18–20.
## Connects To
- **Ch 16**: the undirected models that need sampling in the first place.
- **Ch 18**: sampling as an ingredient in partition-function estimation.
- **Ch 20**: sampling as generation.

69
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch18-partition-function.md Normal file
View file

@ -0,0 +1,69 @@
# Chapter 18: Confronting the Partition Function
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/partition.html
## Core Idea
For undirected models, the log-likelihood gradient splits into a positive phase (push energy down
on data) and a negative phase (push energy up everywhere the model believes) — and the negative
phase requires samples from the model itself. Every technique in this chapter is a way to afford
that negative phase, or to avoid needing Z at all.
## Frameworks Introduced
- **Positive and negative phase**: the gradient of log Z is an expectation under the *model*.
This is the structural reason undirected models are expensive.
- **Contrastive divergence (CD-k)**: initialize the negative chain at the data and run k steps.
Fast; biased; produces spurious modes in regions the short chain never visits.
- **Stochastic maximum likelihood / persistent CD (PCD)**: keep the chain's state across parameter
updates so it has effectively run for a long time. Better mixing, at the cost of a chain that
can fall behind fast-moving parameters.
- **Pseudolikelihood**: replace the joint with a product of conditionals — Z cancels. Cheap; a
different objective, so it optimizes for a different thing.
- **Score matching**: match ∇_x log p instead of p, which eliminates Z because the gradient of
log Z with respect to x is zero. **Ratio matching** and **denoising score matching** are the
variants.
- **Noise-contrastive estimation (NCE)**: turn density estimation into a classification problem —
real data versus noise — and treat the normalizer as a learned parameter.
- **Annealed importance sampling (AIS)** and **bridge sampling**: estimate Z itself, mainly for
evaluation.
## Mental Models
- Read the whole chapter as **four escape routes from Z**: sample it (CD/PCD), sidestep it
(pseudolikelihood, score matching), learn around it (NCE), or estimate it for evaluation only
(AIS).
- Treat score matching's trick — **differentiate with respect to x, not θ, so Z vanishes** — as
the single most consequential idea in Part III for modern practice.
- Expect **spurious modes** from short-chain training; they are a predictable artifact, not a bug
in your code.
## Anti-patterns
- **Comparing likelihoods across undirected models without estimating Z** — the numbers are not
comparable.
- **Using CD-1 and reporting it as maximum likelihood.**
## What changed after 2016
Score matching's downstream career is the story here. Denoising score matching plus a noise
schedule became score-based generative modelling (Song & Ermon 2019) and DDPM (Ho et al. 2020) —
the diffusion family that now dominates image, audio and video generation. NCE became the
backbone of contrastive representation learning (word2vec's negative sampling, then InfoNCE and
CLIP). AIS remains the standard tool for evaluating likelihoods when Z is unknown. Contrastive
divergence and PCD are largely historical. **Confidence: high.**
**This is the chapter whose ideas travelled furthest — read it even if you never train a
Boltzmann machine.**
## Key Takeaways
1. When a normalizer blocks you, ask which of the four escape routes fits your objective.
2. Learn score matching properly; diffusion models are unintelligible without it.
3. Never compare unnormalized likelihoods.
## Connects To
- **Ch 17**: the sampling machinery the negative phase depends on.
- **Ch 14**: denoising autoencoders as score estimators.
- **Ch 20**: the generative models built from these objectives.

63
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch19-approximate-inference.md Normal file
View file

@ -0,0 +1,63 @@
# Chapter 19: Approximate Inference
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/inference.html
## Core Idea
Computing p(h|x) — the posterior over latents — is intractable for most interesting models. The
fix is to reframe inference as optimization: choose a tractable family q, and maximize a lower
bound on the log-likelihood. The gap between the bound and the truth is exactly a KL divergence.
## Frameworks Introduced
- **Inference as optimization**: pick q(h) to maximize the **evidence lower bound (ELBO)**,
L(q) = log p(x) − D_KL(q(h) ‖ p(h|x)). Maximizing L both fits the model and tightens the bound.
- **Expectation maximization (EM)**: alternate between setting q to the current posterior
(E-step) and maximizing with respect to parameters (M-step).
- **MAP inference and sparse coding**: taking a point estimate of h is a degenerate q (a Dirac);
sparse coding's inference step is exactly MAP inference.
- **Variational inference and mean field**: restrict q to a factorized family, q(h) = ∏ q(hᵢ),
and derive fixed-point updates. Tractability is bought with an independence assumption.
- **Reverse-KL consequences**: the ELBO uses D(q‖p), which is mode-seeking — variational
posteriors are characteristically **too narrow**, and underestimate uncertainty.
- **Learned approximate inference / amortization**: train a network to output q's parameters
directly, replacing per-example optimization with one forward pass.
## Mental Models
- Read the ELBO as **"log-likelihood minus the cost of your approximation"** — improving q and
improving the model are the same optimization.
- Expect **underestimated variance** from mean-field posteriors, and never report variational
uncertainty as calibrated without checking.
- Treat amortization as the **encoder** of Chapter 14: the VAE is exactly this idea plus the
reparameterization trick.
## Anti-patterns
- **Reporting a mean-field posterior's credible intervals as if they were exact.**
- **Blaming the model for a poor fit** that is actually a too-restrictive q — diagnose the bound
before the model.
## What changed after 2016
Amortized variational inference became routine (VAEs and descendants); normalizing flows and
importance-weighted bounds (IWAE) gave tighter, more expressive posteriors than mean field. But
the strategic picture changed more: modern large generative models largely **avoid latent-variable
posteriors altogether** — autoregressive transformers have no posterior to infer, and diffusion
models use a fixed forward process, so their "inference" is trivial by construction. Variational
inference remains central in Bayesian deep learning and in structured latent-variable modelling.
**Confidence: high.**
## Key Takeaways
1. Write the ELBO down for any latent-variable model you train; it tells you what you are
actually optimizing.
2. Assume a mean-field posterior is over-confident until proven otherwise.
3. Ask whether your problem needs a latent posterior at all — many modern designs are structured
to avoid one.
## Connects To
- **Ch 13 / Ch 14**: the latent-variable models needing inference, and amortization.
- **Ch 20**: the VAE, assembled from this chapter plus the reparameterization trick.
- **Ch 3**: the KL asymmetry that dictates the narrow-posterior failure mode.

67
engineering/deep-learning-book/skills/deep-learning-book/chapters/ch20-deep-generative-models.md Normal file
View file

@ -0,0 +1,67 @@
# Chapter 20: Deep Generative Models
**Source chapter (free, official):** https://www.deeplearningbook.org/contents/generative_models.html
## Core Idea
The book's capstone: every generative model family circa 2016, organized by how it handles the
intractable quantity. Boltzmann machines and their deep variants pay for the partition function;
VAEs bound the likelihood; GANs avoid likelihood entirely; autoregressive models factor it away.
That taxonomy — **not** the specific models — is what still holds.
## Frameworks Introduced
- **Boltzmann machines, RBMs, deep belief networks, deep Boltzmann machines**: energy-based
models trained with the Ch 18 machinery. Historically pivotal; now largely superseded.
- **Variational autoencoder (VAE)**: an encoder produces q(h|x), a decoder produces p(x|h), and
the **reparameterization trick** makes the sampling step differentiable so the ELBO can be
optimized by backprop. Blurry samples are the predictable consequence of the likelihood
objective plus a limited posterior family.
- **Generative adversarial networks (GANs)**: a generator and a discriminator in a minimax game;
no explicit likelihood, sharp samples, unstable training, and mode collapse as the
characteristic failure.
- **Autoregressive / fully-visible belief networks**: factor p(x) by the chain rule and model each
conditional. Exact likelihood, sequential sampling. (NADE, PixelRNN/PixelCNN, WaveNet era.)
- **Generative stochastic networks; denoising-based generation**: learning a transition operator
rather than a distribution — the direct ancestor of diffusion.
- **Evaluating generative models**: the chapter's warning that likelihood, sample quality and
downstream usefulness are **three different axes** that routinely disagree.
## Mental Models
- Classify any new generative model by **what it does about the intractable term**: bound it,
avoid it, factor it away, or learn a sampler directly. New families are new answers to that one
question.
- Read blurriness vs mode collapse as **the KL asymmetry from Ch 3 made visible**: likelihood-based
models cover modes and blur; adversarial models sharpen and drop modes.
- Never accept a **single** evaluation number for a generative model.
## Anti-patterns
- **Comparing FID across papers with different preprocessing** — the metric is not portable.
- **Treating sample quality as evidence of density estimation quality**, or vice versa.
- **Starting a new project with a Boltzmann machine.**
## What changed after 2016
The most-superseded chapter in the book, and worth reading anyway for its taxonomy.
**Diffusion models** (Ho et al. 2020, built on Ch 18's denoising score matching and Ch 14's
denoising autoencoders) displaced GANs for image, audio and video generation. **Autoregressive
transformers** became the dominant generative model overall — the chain-rule factorization
described here, scaled. VAEs persist mainly as latent-space compressors inside latent diffusion
pipelines rather than as end-user generators. GANs remain useful for fast, low-step generation.
Boltzmann machines are history. Evaluation remains unsolved, exactly as the chapter warned.
**Confidence: high.**
## Key Takeaways
1. Use the taxonomy, not the model list — it classifies architectures invented since.
2. Expect blurring from likelihood objectives and mode-dropping from adversarial ones; pick your
failure mode deliberately.
3. Evaluate generative models on at least likelihood, sample quality, and downstream use.
## Connects To
- **Ch 18**: score matching, whose descendants replaced most of this chapter.
- **Ch 19**: the ELBO the VAE optimizes.
- **references/book_to_2026_delta.md**: the diffusion and autoregressive-transformer lines.

73
engineering/deep-learning-book/skills/deep-learning-book/cheatsheet.md Normal file
View file

@ -0,0 +1,73 @@
# Cheatsheet
Decision rules and thresholds. One line each; the chapter has the reasoning.
## Loss selection (ch03, ch06)
| Target | Distribution | Output unit | Loss |
|---|---|---|---|
| Real, unbounded | Gaussian | linear | MSE |
| Binary | Bernoulli | sigmoid | binary cross-entropy |
| One of K | Categorical | softmax | cross-entropy |
| Real, heavy-tailed | Laplace | linear | MAE |
| Multimodal real | Mixture | mixture density | NLL of the mixture |
Never pair sigmoid output with MSE — the gradient vanishes where the model is most wrong.
## Underfit vs overfit (ch05, ch11)
| Train error | Val error | Verdict | First action |
|---|---|---|---|
| High | High | Underfit / optimization | Check LR, init, capacity — **not** more data |
| Low | High | Overfit | More data → augmentation → regularization |
| Low | Low | Working | Stop; check the metric is the right one |
| ~0 on 20 examples: fails | — | Bug | Fix the pipeline before anything else |
Past the interpolation threshold, prefer more data / longer training / more regularization over
shrinking the model (double descent, ch05).
## Optimizer defaults (ch08)
| Situation | Reach for |
|---|---|
| Any starting point | AdamW, LR ~1e-3 (small nets) / ~1e-4 (large), warmup + cosine decay |
| Recurrence in the graph | Add gradient clipping, always |
| Tiny batches | Group/layer norm, not batch norm |
| Loss flat, grad norm large | Ill-conditioning — momentum, adaptive, or normalization |
| Loss NaN | Numerics first: log(0), div by ~0, exp overflow, exploding grad |
## Regularization ladder (ch07)
More real data → label-preserving augmentation → early stopping → decoupled weight decay →
label smoothing → dropout → smaller model. One at a time; measure the gap after each.
## Architecture prior (ch09, ch10)
| Data | Prior that fits | Note |
|---|---|---|
| Grid, translation-invariant statistics | Convolution | Cheapest when data is limited |
| Sequence, long-range dependence | Attention/transformer | The 2016 RNN advice is superseded |
| Sequence, very long context | State-space / linear recurrence | Post-2016; ch10's gradient analysis applies |
| Arbitrary tabular columns | Neither | Locality prior is false |
## Generative model taxonomy (ch20)
| Family | Handles Z by | Characteristic failure |
|---|---|---|
| Autoregressive | Chain rule — no Z | Slow sequential sampling |
| VAE | Bounding likelihood (ELBO) | Blurry samples |
| GAN | Avoiding likelihood | Mode collapse, unstable training |
| Diffusion / score-based | Learning ∇ log p | Many sampling steps (mitigable) |
| Boltzmann machines | Sampling the negative phase | Historical; do not start here |
## Numerical hygiene (ch04)
Log-space for probability products · stabilized/fused softmax and cross-entropy · clip gradients
with recurrence · bf16 over fp16 when range matters · check condition number before blaming LR.
## Study-order rule (ch01–ch20)
Ch 5 and Ch 11 are the highest-value chapters for a practitioner and can be read early.
Ch 2–4 are prerequisites for Ch 8 specifically. Ch 13→14→19→20 is the only strict chain in
Part III. Ch 18 is worth reading even if you never train an undirected model — score matching
became diffusion.

58
engineering/deep-learning-book/skills/deep-learning-book/glossary.md Normal file
View file

@ -0,0 +1,58 @@
# Glossary
Terms as this skill uses them, with the chapter that develops each. Definitions are written for
lookup, not for teaching — read the chapter file for the reasoning.
- **Adam / AdamW** (ch08, ch07) — adaptive optimizer combining momentum with per-parameter scaling. AdamW decouples weight decay from the gradient, which Adam+L2 does not do correctly.
- **Amortized inference** (ch19, ch14) — training a network to predict a posterior's parameters instead of optimizing per example.
- **Ancestral sampling** (ch16) — sampling a directed model by drawing each variable after its parents.
- **Attention** (ch10) — a learned weighted read over a set of positions; introduced here as a fix for the seq2seq bottleneck, later the whole architecture.
- **Backpropagation** (ch06) — reverse-mode automatic differentiation over the computation graph.
- **Batch normalization** (ch08) — reparameterization normalizing layer inputs by batch statistics.
- **Bias–variance decomposition** (ch05) — expected error as bias² + variance + irreducible noise.
- **Capacity, effective vs representational** (ch05) — functions reachable by the optimizer vs functions expressible by the architecture.
- **Condition number** (ch02, ch04) — ratio of largest to smallest singular value; how amplified small perturbations become.
- **Contrastive divergence (CD-k)** (ch18) — biased approximation of the negative phase using a short chain started at the data.
- **Cross-entropy** (ch03) — H(p,q) = H(p) + D_KL(p‖q); minimizing it over q minimizes KL.
- **Curse of dimensionality** (ch05, ch01) — configurations grow exponentially with dimension, defeating local-interpolation methods.
- **Denoising autoencoder (DAE)** (ch14) — reconstructs clean input from a corrupted copy; its learned map estimates the score.
- **Disentangled representation** (ch15) — separate generative factors on separate directions.
- **Distributed representation** (ch15) — n features describing exponentially many regions, versus one-hot's n.
- **d-separation** (ch16) — reading conditional independence off a directed graph.
- **Double descent** (ch05) — test error falling again past the interpolation threshold; post-dates the book and qualifies its U-curve.
- **ELBO** (ch19) — evidence lower bound; log p(x) minus KL(q ‖ posterior).
- **Empirical risk minimization** (ch08) — minimizing training loss as a surrogate for unmeasurable true risk.
- **Energy-based model** (ch16) — p(x) ∝ exp(−E(x)).
- **Equivariance** (ch09) — a shift in input produces the corresponding shift in output; convolution's defining property.
- **Exposure bias** (ch10) — train/inference mismatch from teacher forcing.
- **Gradient clipping** (ch08, ch10) — rescaling gradients above a norm threshold to survive cliffs.
- **Ill-conditioning** (ch04, ch08) — wide Hessian eigenvalue spread; loss stalls while gradient norm stays large.
- **Importance sampling** (ch17) — sampling from q and reweighting by p/q.
- **KL divergence** (ch03) — asymmetric divergence between distributions; direction decides mode-covering vs mode-seeking.
- **Manifold hypothesis** (ch05, ch14) — real data concentrates near a low-dimensional manifold.
- **Maximum likelihood estimation (MLE)** (ch05) — parameters maximizing data probability; the justification for standard losses.
- **Mean field** (ch19) — factorized variational family; buys tractability with an independence assumption.
- **Mixing time / burn-in** (ch17) — steps an MCMC chain needs before its samples are usable.
- **Mode collapse** (ch20) — a generator covering only part of the data distribution.
- **Momentum / Nesterov momentum** (ch08) — velocity accumulation across gradient steps.
- **No Free Lunch theorem** (ch05) — averaged over all distributions, all learners tie; priors carry all leverage.
- **Noise-contrastive estimation (NCE)** (ch18) — density estimation recast as data-versus-noise classification.
- **Partition function (Z)** (ch16, ch18) — the normalizer of an undirected model; the central intractability of Part III.
- **Persistent contrastive divergence (PCD)** (ch18) — CD with a chain persisted across updates.
- **Pooling** (ch09) — local aggregation giving approximate invariance to small translations.
- **Positive / negative phase** (ch18) — the two terms of an undirected model's likelihood gradient.
- **Pseudolikelihood** (ch18) — product of conditionals, in which Z cancels.
- **Receptive field** (ch09) — the input region influencing one output unit.
- **Regularization** (ch07, ch05) — any change intended to reduce generalization error but not training error.
- **Reparameterization trick** (ch20) — expressing a sample as a differentiable function of noise, making the ELBO backprop-able.
- **Representation learning** (ch15, ch01) — learning features rather than hand-designing them.
- **Saddle point** (ch08) — critical point that is a minimum in some directions and a maximum in others; the dominant critical point in high dimensions.
- **Score matching** (ch18) — matching ∇_x log p instead of p, so Z vanishes; the basis of diffusion models.
- **SGD, minibatch** (ch08) — gradient descent on noisy gradients from data subsets.
- **Singular value decomposition (SVD)** (ch02) — A = U D Vᵀ for any matrix.
- **Sparse coding** (ch13) — heavy-tailed latent prior with inference by optimization.
- **Teacher forcing** (ch10) — training an autoregressive model on ground-truth history.
- **Universal approximation theorem** (ch06) — one wide hidden layer suffices in principle; silent on width, learnability and sample cost.
- **Vanishing / exploding gradients** (ch10, ch08) — repeated Jacobian multiplication shrinking or growing gradients through depth or time.
- **Variational autoencoder (VAE)** (ch20, ch19) — amortized variational inference plus the reparameterization trick.
- **Weight decay** (ch07) — pulling parameters toward zero; equivalent to L2 for SGD, not for Adam.

92
engineering/deep-learning-book/skills/deep-learning-book/patterns.md Normal file
View file

@ -0,0 +1,92 @@
# Patterns
Recurring techniques from the book, stated as instruments with their trade-offs. Each names the
chapter that develops it.
## Derive the loss from the output distribution (ch03, ch06)
Choose what p(y|x) is, then take its negative log. Gaussian → MSE, Bernoulli → binary
cross-entropy, categorical → cross-entropy, Laplace → MAE, mixture → mixture density loss.
**Trade-off:** none — this is strictly better than picking a loss by habit. **Failure mode it
prevents:** MSE on bounded, count-valued or heavily skewed targets.
## Work in log-space (ch04)
Sums of logs instead of products of probabilities; stabilized softmax and `log_softmax` rather
than hand-rolled exp/normalize. **Trade-off:** slightly more care at implementation time.
**Prevents:** silent underflow, NaN losses, and inf logits.
## Train-error-first triage (ch11, ch05)
Read training error before deciding anything. High training error → capacity or optimization is
the bottleneck; more data will not help. Low training error with high validation error → data or
regularization. **Trade-off:** requires honest, non-leaky splits. **Prevents:** the most expensive
common mistake, which is collecting data to fix underfitting.
## Overfit a tiny subset as a smoke test (ch11)
Take 10–50 examples and drive training loss to ~0. If you cannot, you have a bug — not a hard
problem. **Trade-off:** minutes. **Prevents:** weeks of tuning around a broken data pipeline,
a wrong loss reduction, or a detached gradient.
## Random search over grid search (ch11)
With more than about two hyperparameters, random search finds better configurations for the same
budget because it does not spend trials re-testing unimportant dimensions. **Trade-off:** results
are less tidy to tabulate. **Prevents:** exponential waste in dimensions that do not matter.
## Regularize in cost order (ch07)
More real data → augmentation with label-preserving transforms → early stopping → weight decay →
dropout → architecture change. Add one at a time and measure the train/val gap after each.
**Trade-off:** slower than stacking everything. **Prevents:** an unattributable result you cannot
tune.
## Parameter sharing over parameter penalties (ch07, ch09)
When a real invariance exists, encode it structurally (convolution, weight tying) rather than
penalizing a free parameter into behaving. **Trade-off:** the prior is hard — wrong invariance
means a wrong model, with no way for data to override it. **Prevents:** paying for capacity you
then have to regularize away.
## Diagnose by gradient behaviour (ch08, ch04)
Gradient norm exploding → clip. Norm large, loss flat → ill-conditioning; use momentum, an
adaptive optimizer, or normalization. Norm near zero with high loss → saturation or dead units;
check initialization and activations. Loss NaN → numerics before modelling. **Trade-off:**
requires instrumentation. **Prevents:** changing the architecture to fix an optimizer problem.
## Tune learning rate and schedule before optimizer family (ch08)
The schedule usually dominates the choice among SGD/Adam variants. **Trade-off:** none. **Prevents:**
optimizer-shopping while the real problem is a step size two orders of magnitude off.
## Name what prevents the identity map (ch14)
Every autoencoder needs an answer: bottleneck, sparsity penalty, input corruption, or Jacobian
contraction. **Trade-off:** each constraint defines a different notion of useful.
**Prevents:** an overcomplete autoencoder that learns a copy and reports a low loss.
## Escape the partition function deliberately (ch18, ch16)
Four routes: sample the negative phase (CD/PCD), sidestep Z algebraically (pseudolikelihood,
score matching), learn around it (NCE), or estimate it for evaluation only (AIS). **Trade-off:**
each optimizes a different objective, so they are not interchangeable. **Prevents:** discovering
mid-project that your undirected model cannot be trained or compared.
## Pick your generative failure mode (ch20, ch03)
Likelihood-based objectives cover modes and blur. Adversarial objectives sharpen and drop modes.
This follows from KL direction, so it is a design choice, not bad luck. **Trade-off:** you must
decide which error your application tolerates. **Prevents:** treating blurriness as a bug to be
tuned away.
## Evaluate representations by transfer, not reconstruction (ch15, ch14)
Linear probes and downstream task performance. **Trade-off:** needs a downstream task.
**Prevents:** optimizing reconstruction error into a code that memorized the input.
## Report Monte Carlo estimates with error bars (ch17)
Standard error and a mixing diagnostic, always. **Trade-off:** more reporting.
**Prevents:** a confidently unimodal answer from a chain that never left its starting mode.

101
engineering/deep-learning-book/skills/deep-learning-book/references/book_to_2026_delta.md Normal file
View file

@ -0,0 +1,101 @@
# The 2016 → 2026 Delta
The book was published in 2016. *Attention Is All You Need* appeared in 2017. Everything below
tracks what a reader should hold, discount, or replace — per chapter, with a confidence level on
each claim.
Confidence levels: 🟢 well-established and replicated · 🟡 supported but contested or
context-dependent · 🔴 folklore, named as folklore.
---
## Chapters that aged well (read as current)
| Ch | Why it holds |
|---|---|
| 2, 3, 4 | Mathematics does not age. 🟢 |
| 5 | The learning-theory frame is current — with one correction (double descent, below). 🟢 |
| 11 | The practical workflow became the backbone of MLOps practice. 🟢 |
| 16 | The directed/undirected taxonomy classifies models invented since, including transformers as chain-rule factorizations. 🟢 |
| 18 | Score matching became diffusion; NCE became contrastive learning. The chapter's ideas travelled further than its models. 🟢 |
## The five corrections that matter
### 1. Double descent qualifies the U-curve (ch05)
The classical capacity curve — error falls, then rises past the optimum — is incomplete. Past the
interpolation threshold, test error can descend a second time. Belkin et al. (2019) named it;
Nakkiran et al. (2020) showed model-wise, epoch-wise and sample-wise variants in deep networks.
The bias–variance algebra is unchanged; the *advice* "shrink the model when it overfits" is no
longer the only correct move. 🟢
### 2. AdamW: weight decay ≠ L2 under adaptive optimizers (ch07, ch08)
Adding an L2 term to the loss and applying weight decay to the update are equivalent for plain
SGD and **not** equivalent for Adam, because the adaptive denominator rescales the penalty.
Loshchilov & Hutter (2017/2019) decoupled them; AdamW is now the default. The book's Ch 7 treats
the two as interchangeable. 🟢
### 3. Transformers displaced recurrence (ch10, ch12)
Vaswani et al. (2017) removed recurrence entirely: attention gives an O(1) path between any two
positions and parallelizes over sequence length. What survives from Ch 10: the vanishing/exploding
gradient analysis, gradient clipping, teacher forcing, exposure bias. What is superseded: the
architecture recommendation. Note the return of linear-time recurrence in state-space models
(S4, 2021; Mamba, 2023) for long context — which makes Ch 10's analysis live again. 🟢
### 4. Diffusion displaced the Part III generative models (ch14, ch18, ch20)
The line runs directly through the book: denoising autoencoders (Ch 14) → denoising score
matching (Ch 18) → score-based generative models (Song & Ermon, 2019) → DDPM (Ho et al., 2020).
Diffusion now dominates image, audio and video generation; autoregressive transformers dominate
text. VAEs survive mainly as latent-space compressors inside latent-diffusion pipelines. GANs
remain useful for few-step generation. Boltzmann machines are historical. 🟢
### 5. Self-supervised learning vindicated Ch 15 while replacing its methods
The chapter's bet — that unsupervised representation learning would matter — was right, and every
specific method it lists was replaced. Contrastive (SimCLR, MoCo, CLIP), masked prediction (BERT,
MAE) and next-token prediction at scale are the modern routes. Two corrections: greedy layer-wise
pretraining is now purely historical 🟢, and unsupervised disentanglement was shown impossible
without inductive biases or supervision (Locatello et al., 2019) 🟢.
## Additions the book has no chapter for
| Topic | Status |
|---|---|
| **Neural scaling laws** (Kaplan 2020; Hoffmann 2022) | Quantifies Ch 1's "scale matters" and Ch 11's data-vs-capacity decision. 🟢 |
| **Normalization placement** — pre-norm vs post-norm residual, LayerNorm/RMSNorm over BatchNorm | Standard for deep sequence stacks. 🟢 |
| **Warmup + cosine decay schedules** | Standard for transformer training. 🟢 |
| **Calibration** (Guo et al., 2017) | Modern networks are systematically overconfident; temperature scaling is the cheap fix. Ch 3 does not mention it. 🟢 |
| **RLHF / DPO alignment** | Entirely outside the book. 🟢 |
| **Mixture-of-Experts** | The mature form of Ch 12's conditional computation. 🟢 |
| **Mixed precision (fp16 loss scaling, bf16)** | Made Ch 4's numerics an operational daily concern. 🟢 |
| **Lottery-ticket / pruning theory** (Frankle & Carbin, 2019) | Active; the strong form remains contested. 🟡 |
| **"Batch norm works by reducing internal covariate shift"** | The original explanation; challenged by Santurkar et al. (2018), who attribute the effect to smoothing the loss landscape. Treat the mechanism as unsettled. 🟡 |
| **"Local minima are the problem in deep nets"** | 🔴 Folklore the book itself corrects: high-dimensional critical points are overwhelmingly saddles (Ch 8). |
| **"You need a GPU cluster to learn deep learning"** | 🔴 Folklore. Every mechanism in Parts I–II is observable on a laptop-scale model. |
## How to use this file
When a chapter's advice conflicts with current practice, the conflict is almost always in the
**recommendation**, not the **analysis**. The book explains why things fail; the field has changed
what it reaches for. Keep the diagnosis, replace the prescription.
## Sources
1. Vaswani et al., "Attention Is All You Need," NeurIPS 2017 — arXiv:1706.03762.
2. Loshchilov & Hutter, "Decoupled Weight Decay Regularization," ICLR 2019 — arXiv:1711.05101.
3. Belkin, Hsu, Ma & Mandal, "Reconciling modern machine-learning practice and the classical
bias–variance trade-off," PNAS 116(32), 2019; Nakkiran et al., "Deep Double Descent," ICLR 2020.
4. Ho, Jain & Abbeel, "Denoising Diffusion Probabilistic Models," NeurIPS 2020 — arXiv:2006.11239;
Song & Ermon, "Generative Modeling by Estimating Gradients of the Data Distribution,"
NeurIPS 2019.
5. Kaplan et al., "Scaling Laws for Neural Language Models," 2020 — arXiv:2001.08361;
Hoffmann et al., "Training Compute-Optimal Large Language Models," 2022 — arXiv:2203.15556.
6. Locatello et al., "Challenging Common Assumptions in the Unsupervised Learning of Disentangled
Representations," ICML 2019 (best paper) — arXiv:1811.12359.
7. Guo, Pleiss, Sun & Weinberger, "On Calibration of Modern Neural Networks," ICML 2017;
Santurkar et al., "How Does Batch Normalization Help Optimization?," NeurIPS 2018.
8. Dosovitskiy et al., "An Image is Worth 16x16 Words" (ViT), ICLR 2021 — arXiv:2010.11929;
Liu et al., "A ConvNet for the 2020s" (ConvNeXt), CVPR 2022.

93
engineering/deep-learning-book/skills/deep-learning-book/references/prerequisite_map.md Normal file
View file

@ -0,0 +1,93 @@
# Prerequisite Map
The book's part structure is not its dependency structure. Readers who go strictly front-to-back
spend weeks in Chapters 2–4 before touching a network, and a large fraction stop there. This file
gives the actual graph.
---
## The real dependencies
```
ch01 ──────────────────────────────────► (context only; no hard dependents)
ch02 (linear algebra) ──┬──► ch04 ──┬──► ch08 ──► ch09, ch10
│ │
ch03 (probability) ─────┴──► ch05 ──┴──► ch06 ──► ch07 ──► ch11
│
└──► ch13 ──► ch14 ──► ch15
│
ch03 ──► ch16 ──► ch17 ──► ch18 ──► ch19 ──► ch20
```
**Hard prerequisites** (skipping these makes the target chapter unreadable, not merely harder):
| To read | You need | Specifically |
|---|---|---|
| ch08 | ch02, ch04 | eigenvalues, condition number, Hessian, Taylor expansion |
| ch06 | ch03 | the output distributions that determine output units and losses |
| ch07 | ch05, ch04 | bias–variance; KKT for the constraint view of penalties |
| ch18 | ch16, ch17 | the partition function; sampling |
| ch19 | ch03 | KL divergence, and its asymmetry |
| ch20 | ch13, ch19 | the latent-variable template; the ELBO |
**Soft prerequisites** — helpful, not blocking: ch02 before ch13 (PCA), ch09 before ch12,
ch10 before ch12.
## Chapters you can read early, out of order
- **ch05** — the single highest-value chapter for a practitioner. Needs only basic probability.
- **ch11** — readable on day one and immediately actionable. Depends on ch05's vocabulary only.
- **ch01** — context; skim it.
- **ch09** — comprehensible with ch06 alone if you accept the optimization details on faith.
## Chapters you can defer or skip, by goal
| Goal | Skip or defer |
|---|---|
| Applied practitioner shipping models | Part III entirely (ch13–20), except ch14's denoising section |
| Preparing for modern generative modelling | Nothing in Part III — but read ch18 before ch20 |
| Interview / fundamentals refresher | ch12 (dated), ch17–19 (unless the role is probabilistic ML) |
| Understanding transformers | ch10 for the gradient analysis; then leave the book for the 2017 paper |
| Research in probabilistic ML | Everything; Part III is the reason this book has no substitute |
## The Part I wall, and how to get through it
Chapters 2–4 are compressed reference material, not pedagogy. Three viable strategies:
1. **Read-on-demand** (recommended for applied readers): skim ch2–4 once for vocabulary, start at
ch05, and return to a specific section when ch08 uses it. The prerequisite table above tells
you exactly which section.
2. **Front-load** (recommended if you intend to read Part III): work ch2–4 properly with a
separate linear algebra source alongside. Budget 2–3× the page count in time.
3. **Substitute**: use a dedicated linear algebra or probability text for Part I and treat these
chapters purely as a notation reference for the rest of the book.
Strategy 1 is right for most readers and is what `scripts/reading_path_planner.py` recommends by
default. Strategy 2 is right when Part III is the destination — its chapters compound, and gaps
compound with them.
## Time budgeting
Reported reading times vary by an order of magnitude, so treat any figure as a planning
assumption, not a fact. As a planning heuristic used by the reading-path planner: a Part I or
Part III chapter is roughly 1.5–2× the time of a Part II chapter of the same length, because the
derivation density is higher. The planner exposes its per-chapter assumptions in
`--output json` so you can recalibrate them against your own first chapter.
## Sources
1. Goodfellow, Bengio & Courville, *Deep Learning*, MIT Press 2016 — table of contents and part
structure: https://www.deeplearningbook.org/
2. Sweller, van Merriënboer & Paas, "Cognitive Architecture and Instructional Design," *Educational
Psychology Review* 10(3), 1998 — element interactivity and intrinsic load, which is what makes
ch2–4 expensive.
3. Sweller & Cooper, "The Use of Worked Examples as a Substitute for Problem Solving," *Cognition
and Instruction* 2(1), 1985 — the worked-example effect for novices.
4. Bjork & Bjork, "Making Things Hard on Yourself, But in a Good Way: Creating Desirable
Difficulties to Enhance Learning," 2011.
5. Roediger & Karpicke, "Test-Enhanced Learning," *Psychological Science* 17(3), 2006.
6. Ericsson, Krampe & Tesch-Römer, "The Role of Deliberate Practice in the Acquisition of Expert
Performance," *Psychological Review* 100(3), 1993.
7. Chi, Bassok, Lewis, Reimann & Glaser, "Self-Explanations: How Students Study and Use Examples
in Learning to Solve Problems," *Cognitive Science* 13(2), 1989.

80
engineering/deep-learning-book/skills/deep-learning-book/references/rights_and_use.md Normal file
View file

@ -0,0 +1,80 @@
# Why This Is a Companion, Not a Compiled Copy
This repository ships `engineering/book-to-skill`, which compiles a document on disk into a
knowledge-base skill. This skill was **not** produced that way, and the reason is worth stating
plainly, because the same reasoning applies to every future request of the form "turn this book
into a skill."
---
## The three facts that decided it
1. **The book is copyrighted.** *Deep Learning* (Goodfellow, Bengio & Courville) is published by
MIT Press, 2016. Free-to-read is not free-to-redistribute; the two are unrelated.
2. **The publisher's position is explicit.** deeplearningbook.org states that the HTML-only
presentation exists as a deliberate friction against copying and editing, required by the
authors' contract with MIT Press, and that PDF distribution is not permitted.
3. **This repository is public.** `book-to-skill`'s own rights gate refuses a `shareable`
distribution without one of `public-domain`, `open-license`, `internal-docs`, or
`author-permission`. None applies here. Its `references/rights_and_provenance.md` lists
"publish a compiled skill of a copyrighted book to a public marketplace" under **Do not**.
`book-to-skill` also has a hard rule that it converts files already on disk and never scrapes a
book from the web — so the requested pipeline could not have been run against a URL regardless.
## What was built instead
An **original companion**: every chapter file is written from domain knowledge, states what the
chapter establishes, how to use it, and where it has aged, and links to the official free
chapter. The organizing structure — 20 chapters in 3 parts — is the book's published table of
contents, which is factual metadata, not expression.
Concretely, this skill contains:
- **No passages, paragraphs, sentences, or figures from the book.**
- **No paraphrase-per-paragraph** — the compression ratio is roughly a whole chapter to a page,
which is a synthesis, not a substitute.
- **Links to the official chapters**, so the skill sends readers *to* the book rather than
replacing it.
- **Original material the book does not contain**: the 2016→2026 delta layer, the prerequisite
graph, the study method, and four executable tools.
Under the idea/expression line (17 U.S.C. §102(b); *Baker v. Selden*), what this skill carries —
the names of methods, the structure of an argument, decision rules stated plainly, and terms
defined in other words — sits on the ideas side. The chapters themselves, in the authors' prose,
sit on the expression side, and stay at deeplearningbook.org.
**Not legal advice.** This is the posture and its reasoning. Where money or publication is
involved, ask a lawyer.
## The rule this establishes for the repository
> When a user asks to convert a copyrighted work into a shareable skill, build a **companion**
> that indexes and updates the source, not a **compilation** that reproduces it. Compile only
> when the rights gate clears — and keep compiled output local when it does not.
A companion is often the better artifact anyway. A compilation freezes a source at its
publication date; a companion can say which parts of a ten-year-old text are still true, which
is precisely what a reader of a 2016 deep learning book needs most.
## Use this skill correctly
- **Do** read the book at deeplearningbook.org; this navigates it.
- **Do** use the delta reference before acting on any 2016-era recommendation.
- **Don't** treat the chapter files as a substitute for reading the chapters — they are the
answer key for retrieval practice, not the material.
- **Don't** extend this skill by pasting book text into it. That converts a companion into the
thing this file exists to avoid.
## Sources
1. Goodfellow, Bengio & Courville, *Deep Learning*, MIT Press, 2016 — https://www.deeplearningbook.org/
2. deeplearningbook.org — the site's own statement on its HTML-only format and PDF distribution.
3. 17 U.S.C. §102(b) — ideas, procedures, processes and methods of operation are outside
copyright.
4. *Baker v. Selden*, 101 U.S. 99 (1879) — the idea/expression dichotomy.
5. 17 U.S.C. §107 — fair use as a four-factor defence, assessed case by case.
6. *Authors Guild v. Google, Inc.*, 804 F.3d 202 (2d Cir. 2015) — indexing that does not
substitute for the original held transformative.
7. This repository: `engineering/book-to-skill/skills/book-to-skill/references/rights_and_provenance.md`
(the rights gate) and its SKILL.md hard rules 1 and 6.

99
engineering/deep-learning-book/skills/deep-learning-book/references/study_method_canon.md Normal file
View file

@ -0,0 +1,99 @@
# Studying a Hard Technical Text
This book defeats more readers than it teaches, and the reason is rarely ability. It is a
reference text with textbook page counts, high element interactivity, and no exercises. The
methods below are the evidence-based ones that apply specifically to that shape of material.
---
## The five that matter here
### 1. Retrieval practice beats re-reading
Testing yourself on material produces markedly better long-term retention than reviewing it,
even when re-reading feels more productive at the time (Roediger & Karpicke, 2006). For this
book, the cheap version: after each chapter, close it and write the chapter's core idea, three
concepts, and one thing you could not explain. The chapter files in this skill are structured to
be *checked against*, not read first — use them as the answer key.
### 2. Spacing beats massing
Distributed practice outperforms cramming for durable retention (Cepeda et al., 2006). Practical
rule: revisit ch05 and ch11 at one week and one month. They are the chapters whose vocabulary
every later chapter reuses, so spaced review of those two pays across the whole book.
### 3. Worked examples before problem-solving, for novices
For learners without domain schemas, studying worked examples is more effective than solving
equivalent problems — the expertise-reversal effect means this flips once you *have* the schemas
(Sweller & Cooper, 1985; Kalyuga et al., 2003). Consequence for this book: in Ch 8 and Ch 18, work
the derivations the text presents before attempting your own; in Ch 11, do the opposite — you
already have the schema, so run the workflow on a real project.
### 4. Self-explanation
Students who explain each step to themselves while studying examples learn substantially more than
those who do not (Chi et al., 1989). The highest-yield prompt for this book: after every
equation, answer "what would change if this term were removed?" It converts symbol-tracking into
understanding, and it is exactly how the book's own arguments are structured.
### 5. Desirable difficulties
Conditions that slow acquisition often improve retention and transfer (Bjork & Bjork, 2011).
Interleaving Part II chapters with attempts to apply them beats reading Part II straight through,
even though it feels slower. This is the empirical case for the reading-path planner's
"read one chapter, then implement something" cadence.
---
## Managing cognitive load in Chapters 2–4 and 16–20
High **element interactivity** — where symbols cannot be understood one at a time because they
only make sense in relation to each other — is what makes these chapters expensive (Sweller, van
Merriënboer & Paas, 1998). Intrinsic load cannot be reduced by better presentation; it can only be
sequenced. Two consequences:
- **Do not skim a high-interactivity derivation.** Skimming works for prose and fails for
Chapter 18. Either work it or defer it explicitly.
- **Chunk by named object.** "Score matching" as one unit is cheaper than seven equations. Every
chapter file in this skill names its objects for exactly this reason.
## A cadence that works for this book
1. **Skim** the chapter's section headings and the summary. Five minutes.
2. **Read** with a pen, working every derivation you intend to keep.
3. **Close the book** and write, from memory: the core idea, three concepts, one confusion.
4. **Check** against this skill's chapter file — it is deliberately written as a synthesis, so a
mismatch is informative.
5. **Apply** something small before moving on (a five-line implementation, a diagnosis of a real
training run, a re-derivation).
6. **Revisit** at one week for ch05, ch08, ch11, ch18.
Step 5 is the one most readers skip and the one that decides whether any of it survives.
## What not to do
- **Reading front-to-back through Part I as a first pass.** See `prerequisite_map.md`.
- **Highlighting.** It produces a strong feeling of learning and among the weakest measured
effects of the common study techniques (Dunlosky et al., 2013).
- **Taking notes that restate the text.** Restatement is transcription. Write the answer to
"when would I use this?" instead — that is the note that gets read again.
- **Waiting to "finish the math" before touching a model.** The book's own Ch 11 argues against
it: build the end-to-end pipeline early.
## Sources
1. Roediger & Karpicke, "Test-Enhanced Learning: Taking Memory Tests Improves Long-Term
Retention," *Psychological Science* 17(3), 2006.
2. Cepeda, Pashler, Vul, Wixted & Rohrer, "Distributed Practice in Verbal Recall Tasks: A Review
and Quantitative Synthesis," *Psychological Bulletin* 132(3), 2006.
3. Sweller & Cooper, "The Use of Worked Examples as a Substitute for Problem Solving in Learning
Algebra," *Cognition and Instruction* 2(1), 1985; Kalyuga, Ayres, Chandler & Sweller, "The
Expertise Reversal Effect," *Educational Psychologist* 38(1), 2003.
4. Chi, Bassok, Lewis, Reimann & Glaser, "Self-Explanations," *Cognitive Science* 13(2), 1989.
5. Bjork & Bjork, "Making Things Hard on Yourself, But in a Good Way," in *Psychology and the Real
World*, 2011.
6. Sweller, van Merriënboer & Paas, "Cognitive Architecture and Instructional Design,"
*Educational Psychology Review* 10(3), 1998.
7. Dunlosky, Rawson, Marsh, Nathan & Willingham, "Improving Students' Learning With Effective
Learning Techniques," *Psychological Science in the Public Interest* 14(1), 2013.

292
engineering/deep-learning-book/skills/deep-learning-book/scripts/capacity_planner.py Normal file
View file

@ -0,0 +1,292 @@
#!/usr/bin/env python3
"""capacity_planner.py — capacity, the generalization gap, and what to spend next.
Chapter 5 gives the bias-variance budget and the classical U-shaped capacity curve.
Chapter 7 gives the regularizers you spend from that budget. This tool combines them
into one recommendation: given a measured gap and a parameters-per-example ratio, what
is the cheapest next purchase?
It carries one correction the 2016 text predates. The classical advice "shrink the
model when it overfits" assumed the underparameterized side of the U-curve. Past the
interpolation threshold, test error can fall again (double descent; Belkin et al. 2019,
Nakkiran et al. 2020), so in the overparameterized regime this tool ranks *more data,
more regularization, longer training* above *smaller model* and says why.
Standard library only. No network calls.
Exit codes:
0 a plan was produced and the fit looks balanced
1 a plan was produced and an action is recommended
2 usage error (argparse)
4 inputs are inconsistent (e.g. validation error below training error by a wide
margin, which usually means a leaky split rather than a good model)
"""
from __future__ import annotations
import argparse
import json
import sys
# Regularization ladder in cost order (ch07). Cost is effort+risk, not compute.
LADDER = [
("more-real-data", "Collect more real labelled data",
"The only intervention that reduces variance without adding bias. Everything below "
"is a substitute for it.", "ch05, ch07"),
("augmentation", "Label-preserving data augmentation",
"Cheapest substitute for real data where a real invariance exists. Verify the "
"transform preserves the label — a flip destroys the label on digits.", "ch07"),
("early-stopping", "Early stopping on a validation metric",
"Approximately equivalent to L2 under a quadratic approximation, at near-zero cost. "
"Usually the first thing to try.", "ch07"),
("weight-decay", "Decoupled weight decay (AdamW)",
"Damps directions the data does not constrain. Use AdamW, not Adam with an L2 term "
"in the loss — they are not equivalent under an adaptive optimizer.", "ch07, ch08"),
("label-smoothing", "Label smoothing",
"Reduces over-confidence on the training targets; cheap and usually harmless.",
"ch07 (noise injection on labels)"),
("dropout", "Dropout",
"Approximate ensembling. Note the interaction with batch norm, and that modern "
"large stacks regularize far more lightly than 2016 practice.", "ch07"),
("parameter-sharing", "Structural parameter sharing",
"The strongest regularizer: remove parameters rather than penalize them. Only valid "
"when a real invariance exists (convolution, weight tying).", "ch07, ch09"),
("smaller-model", "Reduce model capacity",
"Classical advice from the underparameterized side of the U-curve. In the "
"overparameterized regime this is ranked last, not first.", "ch05"),
]
# Parameters-per-example ratio bands. Approximate regime markers, not a threshold
# theorem — the interpolation threshold depends on the task, the architecture and the
# label noise, and cannot be read off a parameter count alone.
UNDERPARAMETERIZED_MAX = 1.0
OVERPARAMETERIZED_MIN = 10.0
def classify_regime(params: int, examples: int,
under_max: float = UNDERPARAMETERIZED_MAX,
over_min: float = OVERPARAMETERIZED_MIN) -> tuple[str, float, str]:
ratio = params / examples
if ratio < under_max:
regime = "underparameterized"
note = ("Fewer parameters than training examples. The classical U-curve applies "
"directly here: reducing capacity is a legitimate response to overfitting.")
elif ratio < over_min:
regime = "near-interpolation"
note = ("Roughly at the interpolation threshold, where the classical curve peaks "
"and double descent begins. This is the worst place to sit: both more "
"capacity and less capacity can improve test error, so measure rather "
"than reason.")
else:
regime = "overparameterized"
note = ("Many more parameters than examples. The classical 'shrink the model' "
"advice is not reliable here — double descent means a larger model with "
"more data or stronger regularization often generalizes better.")
return regime, ratio, note
def plan(params: int, examples: int, train_error: float, val_error: float,
applied: set[str], target_error: float | None,
overfit_rel_gap: float,
under_max: float = UNDERPARAMETERIZED_MAX,
over_min: float = OVERPARAMETERIZED_MIN) -> dict:
regime, ratio, regime_note = classify_regime(params, examples, under_max, over_min)
denom = max(abs(train_error), 1e-6)
gap = val_error - train_error
rel_gap = gap / denom
if target_error is not None and train_error > target_error * 1.15:
verdict = "UNDERFIT"
headline = ("Training error is above target — capacity or optimization is the "
"bottleneck. More data cannot help yet.")
actions = [
("tune-lr-schedule", "Tune learning rate and schedule first",
"Usually dominates the choice of optimizer family. Warmup plus cosine decay "
"is the modern default for deep stacks.", "ch08"),
("check-init", "Check initialization scale (He / Xavier)",
"Bad initialization is a common silent cause of a model that will not fit.",
"ch08"),
("remove-regularization", "Remove regularization you already added",
"Every regularizer you applied is buying variance reduction you cannot "
"currently afford.", "ch07"),
("add-capacity", "Add capacity (width or depth)",
"Only after the three above — an optimization problem does not respond to "
"more parameters.", "ch05, ch06"),
]
elif rel_gap > overfit_rel_gap:
verdict = "OVERFIT"
headline = (f"Validation error exceeds training error by {rel_gap:.0%} of the "
"training error — spend from the regularization budget.")
ladder = [item for item in LADDER if item[0] not in applied]
if regime == "underparameterized":
actions = ladder
else:
# Push smaller-model to the end and say why.
actions = ([item for item in ladder if item[0] != "smaller-model"]
+ [item for item in ladder if item[0] == "smaller-model"])
else:
verdict = "BALANCED"
headline = ("The gap is within tolerance. Neither more capacity nor more "
"regularization is indicated by these numbers.")
actions = [
("verify-metric", "Verify the metric is the one you care about",
"A balanced fit on the wrong proxy is still the wrong model.", "ch11"),
("check-splits", "Confirm the split is clean and the result holds across seeds",
"A small dataset with one seed is not a measurement.", "ch05, ch11"),
]
return {
"verdict": verdict,
"headline": headline,
"regime": regime,
"regime_note": regime_note,
"params": params,
"train_examples": examples,
"params_per_example": round(ratio, 3),
"train_error": train_error,
"val_error": val_error,
"gap": round(gap, 6),
"relative_gap": round(rel_gap, 4),
"target_error": target_error,
"already_applied": sorted(applied),
"actions": [
{"id": a[0], "action": a[1], "why": a[2], "chapter": a[3]}
for a in actions
],
"thresholds": {
"overfit_rel_gap": overfit_rel_gap,
"underparameterized_max": under_max,
"overparameterized_min": over_min,
},
"double_descent_caveat": (
regime != "underparameterized" and verdict == "OVERFIT"
),
}
def render(result: dict) -> str:
lines = [
"CAPACITY & REGULARIZATION PLAN",
"=" * 70,
f"Verdict : {result['verdict']}",
f" {result['headline']}",
"",
f"Regime : {result['regime']} "
f"({result['params_per_example']} params per training example)",
f" {result['regime_note']}",
"",
f"Train err : {result['train_error']} Val err: {result['val_error']} "
f"gap: {result['gap']} ({result['relative_gap']:.0%} of train error)",
]
if result["already_applied"]:
lines.append(f"Applied : {', '.join(result['already_applied'])} (excluded below)")
lines += ["", "Do these in order:", "-" * 70]
for index, action in enumerate(result["actions"], start=1):
lines.append(f"{index}. {action['action']} [{action['chapter']}]")
lines.append(f" {action['why']}")
lines.append("-" * 70)
if result["double_descent_caveat"]:
lines.append(
"Double-descent caveat: 'reduce capacity' is ranked LAST here because this "
"model is at or past the interpolation threshold, where the classical "
"U-curve advice is unreliable (Belkin 2019, Nakkiran 2020 — both post-date "
"the book). See references/book_to_2026_delta.md."
)
lines.append("Change one thing per experiment and re-measure the gap. (ch11)")
return "\n".join(lines)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description="Turn a measured generalization gap into an ordered action plan.",
epilog="Exit codes: 0 balanced · 1 action recommended · 4 inconsistent inputs.",
)
parser.add_argument("--params", type=int, help="trainable parameter count")
parser.add_argument("--train-examples", type=int, help="number of training examples")
parser.add_argument("--train-error", type=float,
help="training error or loss (same units as --val-error)")
parser.add_argument("--val-error", type=float, help="validation error or loss")
parser.add_argument("--target-error", type=float,
help="the value that would count as success; without it, "
"underfitting cannot be distinguished from convergence")
parser.add_argument("--applied", default="",
help="comma-separated regularizers already in use, from: "
+ ",".join(item[0] for item in LADDER))
parser.add_argument("--overfit-rel-gap", type=float, default=0.30,
help="relative gap above which the fit counts as overfitting "
"(default: 0.30)")
parser.add_argument("--underparameterized-max", type=float,
default=UNDERPARAMETERIZED_MAX,
help="params-per-example below which the classical U-curve "
f"applies directly (default: {UNDERPARAMETERIZED_MAX})")
parser.add_argument("--overparameterized-min", type=float,
default=OVERPARAMETERIZED_MIN,
help="params-per-example above which double descent makes "
"'shrink the model' unreliable "
f"(default: {OVERPARAMETERIZED_MIN}). These are heuristic "
"bands, not a threshold theorem — the interpolation point "
"depends on task, architecture and label noise.")
parser.add_argument("--output", choices=("text", "json"), default="text")
parser.add_argument("--sample", action="store_true",
help="run against a built-in overparameterized example")
args = parser.parse_args(argv)
if args.sample:
args.params, args.train_examples = 12_000_000, 50_000
args.train_error, args.val_error = 0.01, 0.22
args.applied = "early-stopping"
required = (args.params, args.train_examples, args.train_error, args.val_error)
if any(value is None for value in required):
parser.error("--params, --train-examples, --train-error and --val-error are all "
"required (or use --sample)")
if args.params <= 0 or args.train_examples <= 0:
parser.error("--params and --train-examples must be positive")
if args.underparameterized_max >= args.overparameterized_min:
# Overlapping bands silently mis-class the regime, and the cost is not
# cosmetic: an overparameterized model reported as underparameterized ranks
# "shrink the model" FIRST, inverting the double-descent correction this
# tool exists to apply.
parser.error(
f"--underparameterized-max ({args.underparameterized_max}) must be less "
f"than --overparameterized-min ({args.overparameterized_min}); the bands "
"are ordered and must not overlap"
)
known = {item[0] for item in LADDER}
applied = {token.strip() for token in args.applied.split(",") if token.strip()}
unknown = applied - known
if unknown:
parser.error(f"unknown --applied value(s): {', '.join(sorted(unknown))}; "
f"choose from {', '.join(sorted(known))}")
if args.val_error < args.train_error - 0.05 * max(abs(args.train_error), 1e-6):
payload = {
"status": "inconsistent_input",
"reason": "validation error is materially below training error",
"explanation": "This usually means a leaky split, a validation set that is "
"easier than the training set, or regularization active at "
"train time but not at eval (dropout, augmentation). Fix the "
"measurement before acting on it.",
"chapter": "ch05, ch11",
}
if args.output == "json":
print(json.dumps(payload, indent=2))
else:
print("INCONSISTENT INPUT — " + payload["reason"])
print(payload["explanation"])
return 4
result = plan(args.params, args.train_examples, args.train_error, args.val_error,
applied, args.target_error, args.overfit_rel_gap,
args.underparameterized_max, args.overparameterized_min)
if args.output == "json":
print(json.dumps(result, indent=2))
else:
print(render(result))
return 0 if result["verdict"] == "BALANCED" else 1
if __name__ == "__main__":
sys.exit(main())

420
engineering/deep-learning-book/skills/deep-learning-book/scripts/model_arithmetic.py Normal file
View file

@ -0,0 +1,420 @@
#!/usr/bin/env python3
"""model_arithmetic.py — parameters, FLOPs and activation memory for a layer stack.
Chapter 6 makes the point that the real cost of depth in training is activation memory:
the backward pass needs the forward activations, which is why gradient checkpointing
trades compute for memory. Chapter 9 makes the receptive-field and parameter-sharing
arithmetic explicit. This tool does that arithmetic for a declared stack, and — the part
that catches real bugs — refuses to continue when one layer's output shape cannot feed
the next.
Shapes are per example, excluding the batch dimension. FLOPs are per example, per
forward pass; a training step costs roughly 3x a forward pass (forward + backward).
Layer types: input, linear, conv2d, pool2d, flatten, embedding, layernorm, activation,
dropout, mha (multi-head self-attention), lstm, gru.
A linear layer on a 2-D (seq, features) input is treated as position-wise: one weight
matrix shared across all positions, as in a transformer feedforward block. Parameters do
not scale with sequence length; compute does. Flatten first only when you really mean a
dense layer over the whole flattened sequence — that is a different layer, and its
parameter count is seq_len times larger.
conv2d "same" padding follows TensorFlow/Keras SAME: output is ceil(H / stride), with
any needed padding split across the two sides (and the extra pixel going to the bottom
and right at even kernel sizes). PyTorch's padding='same' is symmetric-only and rejects
a stride other than 1, so a strided PyTorch layer will not match this row — declare it
as "valid" with the padding you actually apply if you need that case exactly.
Standard library only. No frameworks, no network calls.
Exit codes:
0 the stack is consistent and the report was produced
2 usage error (argparse)
4 the spec could not be parsed (bad JSON, unknown layer type, missing field)
5 shape mismatch between consecutive layers — the offending layer is named
"""
from __future__ import annotations
import argparse
import json
import math
import sys
BYTES_PER_ELEMENT = {"fp32": 4, "tf32": 4, "fp16": 2, "bf16": 2, "fp8": 1}
# A multiply-accumulate is counted as 2 FLOPs. --convention mac reports MACs instead.
FLOPS_PER_MAC = 2
class SpecError(ValueError):
"""The spec is malformed: bad JSON, unknown layer, or a missing field."""
class ShapeError(ValueError):
"""A layer cannot consume the shape the previous layer produced."""
def _require(layer: dict, key: str, index: int):
if not isinstance(layer, dict):
raise SpecError(f"layer {index} is {type(layer).__name__}, not an object")
if key not in layer:
raise SpecError(f"layer {index} ({layer.get('type', '?')}) is missing '{key}'")
return layer[key]
def _positive_int(layer: dict, key: str, index: int, default: int | None = None) -> int:
"""Read an integer field, rejecting non-numeric and non-positive values.
Without this, a non-numeric "filters" raised ValueError and a zero "stride" or
"groups" raised ZeroDivisionError — both escaping as tracebacks rather than the
documented exit 4.
"""
raw = layer.get(key, default) if isinstance(layer, dict) else default
if raw is None:
raise SpecError(f"layer {index} ({layer.get('type', '?')}) is missing '{key}'")
try:
value = int(raw)
except (TypeError, ValueError):
raise SpecError(
f"layer {index} ({layer.get('type', '?')}): '{key}' must be an integer, "
f"got {raw!r}"
) from None
if value <= 0:
raise SpecError(
f"layer {index} ({layer.get('type', '?')}): '{key}' must be positive, got {value}"
)
return value
def _prod(shape: tuple[int, ...]) -> int:
total = 1
for dim in shape:
total *= dim
return total
def step(layer: dict, shape: tuple[int, ...], index: int) -> tuple[tuple[int, ...], int, int]:
"""Return (output_shape, parameters, macs) for one layer given its input shape."""
kind = _require(layer, "type", index)
if kind == "input":
raw_shape = _require(layer, "shape", index)
if not isinstance(raw_shape, (list, tuple)) or not raw_shape:
raise SpecError(f"layer {index} (input): 'shape' must be a non-empty list")
dims = []
for dim in raw_shape:
try:
dim = int(dim)
except (TypeError, ValueError):
raise SpecError(
f"layer {index} (input): shape entries must be integers, got {dim!r}"
) from None
if dim <= 0:
raise SpecError(f"layer {index} (input): shape entries must be positive")
dims.append(dim)
return tuple(dims), 0, 0
if kind == "linear":
units = _positive_int(layer, "units", index)
bias = bool(layer.get("bias", True))
if len(shape) == 2:
# Per-token (position-wise) linear over a (seq, features) sequence: one
# weight matrix shared across positions, exactly like a transformer FFN
# projection. Parameters do NOT scale with sequence length; compute does.
# Flattening instead would multiply the parameter count by seq_len, which
# is a different layer and almost never the intended one.
seq, features = shape
params = features * units + (units if bias else 0)
return (seq, units), params, seq * features * units
if len(shape) != 1:
raise ShapeError(
f"layer {index} (linear) needs a 1-D or 2-D input, got {shape}. "
"Insert a flatten layer to collapse a feature map into one vector."
)
params = shape[0] * units + (units if bias else 0)
return (units,), params, shape[0] * units
if kind == "conv2d":
filters = _positive_int(layer, "filters", index)
kernel = _positive_int(layer, "kernel", index)
stride = _positive_int(layer, "stride", index, 1)
padding = layer.get("padding", "same")
if len(shape) != 3:
raise ShapeError(
f"layer {index} (conv2d) needs a 3-D input (C, H, W), got {shape}"
)
channels, height, width = shape
if padding == "same":
# TensorFlow/Keras SAME convention; see the module docstring for how this
# differs from PyTorch's stride-1-only padding='same'.
out_h, out_w = math.ceil(height / stride), math.ceil(width / stride)
elif padding == "valid":
out_h = (height - kernel) // stride + 1
out_w = (width - kernel) // stride + 1
else:
raise SpecError(f"layer {index} (conv2d): padding must be 'same' or 'valid'")
if out_h <= 0 or out_w <= 0:
raise ShapeError(
f"layer {index} (conv2d) with kernel {kernel} and stride {stride} "
f"reduces {height}x{width} to {out_h}x{out_w} — the kernel is larger "
"than the feature map."
)
groups = _positive_int(layer, "groups", index, 1)
if channels % groups or filters % groups:
raise SpecError(
f"layer {index} (conv2d): groups={groups} does not divide "
f"in_channels={channels} and filters={filters}"
)
bias = bool(layer.get("bias", True))
params = (kernel * kernel * (channels // groups) * filters) + (filters if bias else 0)
macs = kernel * kernel * (channels // groups) * filters * out_h * out_w
return (filters, out_h, out_w), params, macs
if kind == "pool2d":
size = _positive_int(layer, "size", index, 2)
stride = _positive_int(layer, "stride", index, size)
if len(shape) != 3:
raise ShapeError(f"layer {index} (pool2d) needs a 3-D input, got {shape}")
channels, height, width = shape
out_h = (height - size) // stride + 1
out_w = (width - size) // stride + 1
if out_h <= 0 or out_w <= 0:
raise ShapeError(
f"layer {index} (pool2d) reduces {height}x{width} to {out_h}x{out_w}"
)
return (channels, out_h, out_w), 0, 0
if kind == "flatten":
return (_prod(shape),), 0, 0
if kind == "embedding":
vocab = _positive_int(layer, "vocab", index)
dim = _positive_int(layer, "dim", index)
seq = _positive_int(layer, "seq_len", index,
default=shape[0] if shape else 1)
return (seq, dim), vocab * dim, 0 # a lookup, not a matmul
if kind == "layernorm":
features = shape[-1]
return shape, 2 * features, 0
if kind in ("activation", "dropout"):
return shape, 0, 0
if kind == "mha":
# Multi-head self-attention over a (seq, d_model) input.
if len(shape) != 2:
raise ShapeError(
f"layer {index} (mha) needs a 2-D input (seq_len, d_model), got {shape}"
)
seq, d_model = shape
heads = _positive_int(layer, "heads", index, 8)
if d_model % heads:
raise SpecError(
f"layer {index} (mha): d_model={d_model} is not divisible by heads={heads}"
)
# 4 projections (Q, K, V, O), each d_model x d_model.
params = 4 * d_model * d_model + (4 * d_model if layer.get("bias", True) else 0)
proj_macs = 4 * seq * d_model * d_model
# Scores (seq x seq x d_model) and the weighted value sum, both quadratic in seq.
attn_macs = 2 * seq * seq * d_model
return shape, params, proj_macs + attn_macs
if kind in ("lstm", "gru"):
if len(shape) != 2:
raise ShapeError(
f"layer {index} ({kind}) needs a 2-D input (seq_len, features), got {shape}"
)
seq, features = shape
units = _positive_int(layer, "units", index)
gates = 4 if kind == "lstm" else 3
params = gates * (features * units + units * units + 2 * units)
macs = seq * gates * (features * units + units * units)
out = (seq, units) if layer.get("return_sequences", True) else (units,)
return out, params, macs
raise SpecError(f"layer {index}: unknown layer type {kind!r}")
def analyse(spec: dict, dtype: str, convention: str) -> dict:
if not isinstance(spec, dict):
raise SpecError(f"spec must be a JSON object, got {type(spec).__name__}")
layers = spec.get("layers")
if not isinstance(layers, list) or not layers:
raise SpecError("spec must contain a non-empty 'layers' list")
for index, layer in enumerate(layers):
if not isinstance(layer, dict):
raise SpecError(f"layer {index} is {type(layer).__name__}, not an object")
if layers[0].get("type") != "input":
raise SpecError("the first layer must be of type 'input'")
width = BYTES_PER_ELEMENT[dtype]
shape: tuple[int, ...] = ()
rows = []
total_params = 0
total_macs = 0
total_activations = 0
for index, layer in enumerate(layers):
shape, params, macs = step(layer, shape, index)
activations = _prod(shape)
total_params += params
total_macs += macs
# The input layer's tensor is not a stored intermediate activation.
if index > 0:
total_activations += activations
rows.append({
"index": index,
"type": layer["type"],
"name": layer.get("name", layer["type"]),
"output_shape": list(shape),
"parameters": params,
"macs": macs,
"flops": macs * FLOPS_PER_MAC,
"activation_elements": activations,
})
compute = total_macs if convention == "mac" else total_macs * FLOPS_PER_MAC
return {
"dtype": dtype,
"convention": convention,
"layers": rows,
"totals": {
"parameters": total_params,
"parameter_bytes": total_params * width,
"forward_macs": total_macs,
"forward_flops": total_macs * FLOPS_PER_MAC,
"reported_compute": compute,
"activation_elements_per_example": total_activations,
"activation_bytes_per_example": total_activations * width,
},
"notes": [
"Shapes and costs are per example; the batch dimension is excluded.",
"A training step costs roughly 3x the forward FLOPs (forward + backward).",
"Activation memory is the training-time cost of depth (ch06). Gradient "
"checkpointing trades compute for it.",
"Optimizer state is extra: Adam/AdamW holds two moments per parameter, so "
"budget ~3x parameter bytes for weights plus state in fp32.",
"Attention cost is quadratic in sequence length — visible in the mha row.",
],
}
SAMPLE_SPEC = {
"name": "small convnet, CIFAR-shaped",
"layers": [
{"type": "input", "shape": [3, 32, 32]},
{"type": "conv2d", "filters": 32, "kernel": 3, "padding": "same"},
{"type": "activation", "name": "relu"},
{"type": "pool2d", "size": 2},
{"type": "conv2d", "filters": 64, "kernel": 3, "padding": "same"},
{"type": "activation", "name": "relu"},
{"type": "pool2d", "size": 2},
{"type": "flatten"},
{"type": "linear", "units": 128},
{"type": "activation", "name": "relu"},
{"type": "dropout"},
{"type": "linear", "units": 10},
],
}
def human(value: int) -> str:
for unit, scale in (("G", 1e9), ("M", 1e6), ("K", 1e3)):
if value >= scale:
return f"{value / scale:.2f}{unit}"
return str(value)
def render(result: dict) -> str:
lines = [
"MODEL ARITHMETIC",
"=" * 78,
f"{'#':>2} {'layer':<12} {'output shape':<18} {'params':>12} "
f"{'MFLOPs':>10} {'act.elts':>10}",
"-" * 78,
]
for row in result["layers"]:
shape = "x".join(str(dim) for dim in row["output_shape"])
lines.append(
f"{row['index']:>2} {row['type']:<12} {shape:<18} "
f"{row['parameters']:>12,} {row['flops'] / 1e6:>10.2f} "
f"{row['activation_elements']:>10,}"
)
totals = result["totals"]
lines += [
"-" * 78,
f"Parameters : {totals['parameters']:,} "
f"({human(totals['parameter_bytes'])}B at {result['dtype']})",
f"Forward FLOPs/ex : {human(totals['forward_flops'])} "
f"({human(totals['forward_macs'])} MACs)",
f"Training step (~3x) : {human(totals['forward_flops'] * 3)} FLOPs/ex",
f"Activations/ex : {totals['activation_elements_per_example']:,} elements "
f"({human(totals['activation_bytes_per_example'])}B at {result['dtype']})",
"",
]
lines.append(f"At batch 128: activations ≈ "
f"{human(totals['activation_bytes_per_example'] * 128)}B")
lines.append("")
for note in result["notes"]:
lines.append(f"· {note}")
return "\n".join(lines)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description="Compute parameters, FLOPs and activation memory for a layer stack.",
epilog="Exit codes: 0 ok · 4 bad spec · 5 shape mismatch.",
)
parser.add_argument("--spec", help="path to a JSON layer spec")
parser.add_argument("--spec-sample", action="store_true",
help="print the built-in sample spec and exit")
parser.add_argument("--dtype", choices=sorted(BYTES_PER_ELEMENT), default="fp32")
parser.add_argument("--convention", choices=("flop", "mac"), default="flop",
help="report FLOPs (1 MAC = 2 FLOPs) or raw MACs")
parser.add_argument("--output", choices=("text", "json"), default="text")
parser.add_argument("--sample", action="store_true",
help="analyse the built-in sample spec")
args = parser.parse_args(argv)
if args.spec_sample:
print(json.dumps(SAMPLE_SPEC, indent=2))
return 0
if args.sample:
spec = SAMPLE_SPEC
elif args.spec:
try:
with open(args.spec, encoding="utf-8") as handle:
spec = json.load(handle)
except OSError as error:
print(f"cannot read spec: {error}", file=sys.stderr)
return 4
except json.JSONDecodeError as error:
print(f"spec is not valid JSON: {error}", file=sys.stderr)
return 4
else:
parser.error("--spec is required (or use --sample / --spec-sample)")
try:
result = analyse(spec, args.dtype, args.convention)
except ShapeError as error:
print(f"SHAPE MISMATCH — {error}", file=sys.stderr)
print("A shape mismatch is a modelling error, not a typing error (ch02).",
file=sys.stderr)
return 5
except SpecError as error:
print(f"BAD SPEC — {error}", file=sys.stderr)
return 4
if args.output == "json":
print(json.dumps(result, indent=2))
else:
print(render(result))
return 0
if __name__ == "__main__":
sys.exit(main())

459
engineering/deep-learning-book/skills/deep-learning-book/scripts/reading_path_planner.py Normal file
View file

@ -0,0 +1,459 @@
#!/usr/bin/env python3
"""reading_path_planner.py — turn a goal into an ordered, prerequisite-closed reading path.
The book's part order is not its dependency order. Read front-to-back and you spend
weeks in Chapters 2-4 before touching a network; skip Part I entirely and Chapter 8
becomes unreadable. This tool takes a goal and a background, resolves the *hard*
prerequisites of the chapters that goal needs, orders them, and prices the result
in weeks at your stated pace.
It refuses two things rather than guessing:
* a goal whose subject the 2016 book does not cover at all (exit 3) — it names what
covers it instead, rather than inventing the book's position on RLHF;
* a goal too vague to route (exit 4) — it prints the questions that would fix it.
Deterministic keyword scoring. Standard library only. No network calls.
Exit codes:
0 a path was produced
2 usage error (argparse)
3 goal is outside the book's scope
4 goal could not be routed — forcing questions printed
"""
from __future__ import annotations
import argparse
import json
import re
import sys
# --------------------------------------------------------------------------- data
# Hard prerequisites only: skipping one makes the target unreadable, not merely
# harder. Soft prerequisites live in references/prerequisite_map.md and are
# deliberately not enforced here.
PREREQS: dict[int, tuple[int, ...]] = {
1: (),
2: (),
3: (),
4: (2,),
5: (3,),
6: (3,),
7: (4, 5),
8: (2, 4),
9: (6,),
10: (6,),
11: (5,),
12: (6,),
13: (5,),
14: (13,),
15: (14,),
16: (3,),
17: (16,),
18: (16, 17),
19: (3,),
20: (13, 19),
}
TITLES: dict[int, str] = {
1: "Introduction",
2: "Linear Algebra",
3: "Probability and Information Theory",
4: "Numerical Computation",
5: "Machine Learning Basics",
6: "Deep Feedforward Networks",
7: "Regularization for Deep Learning",
8: "Optimization for Training Deep Models",
9: "Convolutional Networks",
10: "Sequence Modeling: Recurrent and Recursive Nets",
11: "Practical Methodology",
12: "Applications",
13: "Linear Factor Models",
14: "Autoencoders",
15: "Representation Learning",
16: "Structured Probabilistic Models for Deep Learning",
17: "Monte Carlo Methods",
18: "Confronting the Partition Function",
19: "Approximate Inference",
20: "Deep Generative Models",
}
# Planning assumptions, not measurements. Part I and Part III chapters carry higher
# derivation density per page, which is why they cost more than their length suggests.
# Exposed in --output json so a reader can recalibrate against their own first chapter.
BASE_HOURS: dict[int, float] = {
1: 1.5, 2: 5.0, 3: 6.0, 4: 4.5, 5: 9.0,
6: 6.0, 7: 6.0, 8: 8.0, 9: 6.0, 10: 7.0, 11: 4.0, 12: 5.0,
13: 4.0, 14: 4.5, 15: 5.0, 16: 6.0, 17: 5.0, 18: 7.0, 19: 6.0, 20: 8.0,
}
# Background multipliers on the hour estimate.
BACKGROUNDS: dict[str, tuple[float, str]] = {
"none": (1.6, "little linear algebra or probability — Part I is real work, not review"),
"math": (0.8, "comfortable with linear algebra and probability, new to ML"),
"applied": (1.0, "ships models, wants the theory underneath"),
"research": (0.7, "reads papers in the field; the book is filling gaps"),
}
# Destination lanes. Each is (label, target chapters, note). Scored by keyword hits.
LANES: dict[str, dict] = {
"practitioner": {
"label": "Applied practitioner — train models that work",
"targets": (5, 6, 7, 8, 11),
"keywords": ("train", "training", "practical", "ship", "production", "apply",
"applied", "practitioner", "debug", "tune", "tuning", "improve",
"overfit", "underfit", "hyperparameter", "baseline", "workflow"),
"note": "Chapters 5 and 11 first; they pay off before you finish Part I.",
},
"vision": {
"label": "Computer vision — convolutional models",
"targets": (6, 7, 8, 9, 12),
"keywords": ("vision", "image", "convolution", "convolutional", "cnn", "conv",
"segmentation", "detection", "pooling", "receptive"),
"note": "Ch 9's prior-vs-data argument is what explains ViT's data hunger later.",
},
"sequence": {
"label": "Sequence modelling — recurrence, gradients through time, attention",
"targets": (6, 8, 10, 12),
"keywords": ("sequence", "rnn", "lstm", "gru", "recurrent", "time series",
"timeseries", "nlp", "language", "text", "speech", "translation",
"transformer", "attention"),
"note": "Read Ch 10 for the gradient analysis, not the architecture advice — "
"see references/book_to_2026_delta.md before applying it.",
},
"generative": {
"label": "Generative modelling — the Part III chain",
"targets": (13, 14, 16, 17, 18, 19, 20),
"keywords": ("generative", "vae", "gan", "diffusion", "autoencoder", "sampling",
"mcmc", "boltzmann", "latent", "variational", "elbo", "density",
"score matching", "partition"),
"note": "Ch 18 is the chapter whose ideas became diffusion. Do not skip it to "
"reach Ch 20.",
},
"representation": {
"label": "Representation and self-supervised learning",
"targets": (5, 13, 14, 15),
"keywords": ("representation", "embedding", "feature", "features", "transfer",
"pretrain", "pretraining", "self-supervised", "unsupervised",
"disentangle", "disentangled", "probe"),
"note": "The bet in Ch 15 was right; every method it lists was replaced. "
"Read it with the delta reference open.",
},
"foundations": {
"label": "Mathematical foundations — the machinery under everything else",
"targets": (2, 3, 4, 5),
"keywords": ("math", "mathematics", "foundation", "foundations", "linear algebra",
"probability", "statistics", "theory", "fundamentals", "basics",
"prerequisite", "prerequisites", "refresher", "interview"),
"note": "Pair with a dedicated linear algebra text; Part I is reference "
"material, not pedagogy.",
},
"optimization": {
"label": "Optimization — why training stalls, diverges, or crawls",
"targets": (2, 4, 8, 11),
"keywords": ("optimization", "optimizer", "sgd", "adam", "momentum", "gradient",
"converge", "convergence", "diverge", "learning rate", "saddle",
"initialization", "batch norm", "normalization", "clipping"),
"note": "Ch 8 is unreadable without Ch 2 and 4. This is the one place the "
"prerequisite is genuinely hard.",
},
"complete": {
"label": "Complete read — all twenty chapters",
"targets": tuple(range(1, 21)),
"keywords": ("everything", "whole book", "entire book", "all chapters",
"cover to cover", "complete", "full read", "read the book",
"start to finish"),
"note": "Even here, read Ch 5 and Ch 11 early rather than in numeric order.",
},
}
# Subjects the 2016 book does not cover. Naming these beats improvising its position.
OUT_OF_SCOPE: dict[str, str] = {
"rlhf": "RLHF / preference tuning — published 2017+; nothing in this book covers it.",
"dpo": "Direct preference optimization — 2023; outside the book entirely.",
"llm": "Large language model training and serving — the book predates it; see "
"engineering/llm-cost-optimizer and references/book_to_2026_delta.md.",
"prompt": "Prompting and in-context learning — post-dates the book.",
"agent": "Agentic systems — outside the book; see engineering/agent-harness.",
"mlops": "MLOps tooling and deployment — see engineering-team/senior-ml-engineer.",
"fine-tun": "Fine-tuning of pretrained foundation models — the book's transfer "
"learning section (ch15) is the nearest thing, and it is not the same.",
"lora": "Parameter-efficient fine-tuning (LoRA and relatives) — 2021+.",
"rag": "Retrieval-augmented generation — outside the book.",
"mamba": "State-space models — 2021+; ch10's gradient analysis is the relevant "
"background the book does provide.",
"fairness": "Fairness, bias auditing and model governance — not treated.",
"reinforcement": "Reinforcement learning — mentioned only in passing (ch12).",
}
# Tokens whose real surface forms a word-boundary match would otherwise miss.
# Everything else matches itself, optionally pluralized.
SURFACE_FORMS: dict[str, tuple[str, ...]] = {
"fine-tun": ("fine-tuning", "fine-tune", "fine-tuned", "finetuning", "finetune"),
"prompt": ("prompt", "prompting", "prompts"),
"agent": ("agent", "agents", "agentic"),
}
def _matches(token: str, text: str) -> bool:
"""True when token appears in text as a whole word (optionally pluralized).
Substring matching is wrong here and was a real defect: "rag" appears inside
"storage", "lora" inside "exploratory", "conv" inside "converge", and "text"
inside "context" — each one producing a confident false refusal or a wrong lane.
"""
for form in SURFACE_FORMS.get(token, (token,)):
# Plain -s only. An -es branch collided with unrelated words: "rag" + "es"
# matches the standalone word "rages", so a goal about overfitting was
# refused as out-of-scope RAG work. No token here needs an -es plural —
# every one ending in s/x/z/ch/sh is already plural or non-count — so any
# irregular form belongs in SURFACE_FORMS, spelled out.
if re.search(rf"\b{re.escape(form)}s?\b", text):
return True
return False
# --------------------------------------------------------------------------- logic
def close_prerequisites(targets: tuple[int, ...]) -> list[int]:
"""Return targets plus every hard prerequisite, in ascending chapter order."""
needed: set[int] = set()
stack = list(targets)
while stack:
chapter = stack.pop()
if chapter in needed:
continue
needed.add(chapter)
stack.extend(PREREQS.get(chapter, ()))
return sorted(needed)
def order_path(chapters: list[int]) -> list[int]:
"""Order chapters so every hard prerequisite precedes its dependent.
Among chapters whose prerequisites are already satisfied, the cheapest
high-value chapter goes first: ch05 and ch11 are promoted because their
vocabulary is reused everywhere and they are actionable immediately.
"""
promoted = {5: -2, 11: -1}
remaining = set(chapters)
placed: list[int] = []
while remaining:
ready = [c for c in remaining if all(p in placed for p in PREREQS.get(c, ()))]
if not ready: # unreachable with the current acyclic table; fail loudly if it changes
raise RuntimeError("prerequisite cycle in PREREQS")
ready.sort(key=lambda c: (promoted.get(c, 0), c))
nxt = ready[0]
placed.append(nxt)
remaining.discard(nxt)
return placed
def score_lanes(goal: str) -> list[tuple[str, int]]:
"""Score every lane by keyword hits in the goal text, best first.
Ties are broken by keyword specificity — the lane whose longest matched
keyword is longest wins — because an equal hit count between a generic term
and a discriminating one should not be settled by luck. "train a transformer"
hits `practitioner` on "train" and `sequence` on "transformer", one each; the
longer, more specific match is the one that names the subject. Lane key is the
final tie-break so the ordering stays deterministic.
"""
text = goal.lower()
scored = []
for key, lane in LANES.items():
matched = [kw for kw in lane["keywords"] if _matches(kw, text)]
if matched:
scored.append((key, len(matched), max(len(kw) for kw in matched)))
scored.sort(key=lambda row: (-row[1], -row[2], row[0]))
return [(key, hits) for key, hits, _ in scored]
def out_of_scope_hits(goal: str) -> list[str]:
text = goal.lower()
return [note for token, note in OUT_OF_SCOPE.items() if _matches(token, text)]
def plan(goal: str, background: str, hours_per_week: float,
include_intro: bool) -> dict:
lane_key, _ = score_lanes(goal)[0]
lane = LANES[lane_key]
targets = lane["targets"]
chapters = close_prerequisites(targets)
if include_intro and 1 not in chapters:
# ch01 is context, not a prerequisite of anything, so it never arrives via
# closure — the flag is the only way to reach it outside the complete lane.
# (An earlier form of this filtered ch01 *out*, which was inert: nothing
# depends on ch01, and the one lane that targets it skipped the filter.)
chapters.append(1)
ordered = order_path(chapters)
multiplier, background_note = BACKGROUNDS[background]
entries = []
for chapter in ordered:
hours = round(BASE_HOURS[chapter] * multiplier, 1)
entries.append({
"chapter": chapter,
"title": TITLES[chapter],
"role": "target" if chapter in targets else "prerequisite",
"hours": hours,
"file": f"chapters/ch{chapter:02d}-*.md",
"url": "https://www.deeplearningbook.org/",
})
total_hours = round(sum(e["hours"] for e in entries), 1)
weeks = round(total_hours / hours_per_week, 1) if hours_per_week > 0 else None
skipped = [c for c in range(1, 21) if c not in ordered]
return {
"goal": goal,
"lane": lane_key,
"lane_label": lane["label"],
"background": background,
"background_note": background_note,
"hours_per_week": hours_per_week,
"path": entries,
"total_hours": total_hours,
"estimated_weeks": weeks,
"skipped_chapters": skipped,
"note": lane["note"],
"assumptions": {
"base_hours_per_chapter": BASE_HOURS,
"background_multiplier": multiplier,
"basis": "planning heuristic, not measurement — recalibrate after chapter one",
},
"hard_prerequisites_applied": {
str(c): list(PREREQS[c]) for c in ordered if PREREQS.get(c)
},
}
# --------------------------------------------------------------------------- output
def render(result: dict) -> str:
lines = [
"READING PATH",
"=" * 64,
f"Goal : {result['goal']}",
f"Lane : {result['lane_label']}",
f"Background : {result['background']} — {result['background_note']}",
f"Budget : {result['total_hours']} h at {result['hours_per_week']} h/week"
f" ≈ {result['estimated_weeks']} weeks",
"",
f"{'#':>3} {'ch':>4} {'hrs':>5} role title",
"-" * 64,
]
for index, entry in enumerate(result["path"], start=1):
lines.append(
f"{index:>3} ch{entry['chapter']:02d} {entry['hours']:>5} "
f"{entry['role']:<12} {entry['title']}"
)
lines.append("-" * 64)
if result["hard_prerequisites_applied"]:
lines.append("Hard prerequisites pulled in:")
for chapter, prereqs in result["hard_prerequisites_applied"].items():
names = ", ".join(f"ch{p:02d}" for p in prereqs)
lines.append(f" ch{int(chapter):02d} needs {names}")
if result["skipped_chapters"]:
skipped = ", ".join(f"ch{c:02d}" for c in result["skipped_chapters"])
lines.append(f"Not in this path: {skipped}")
lines.append("")
lines.append(f"Note: {result['note']}")
lines.append("Hours are a planning heuristic. Recalibrate after your first chapter.")
lines.append("Read the chapters free at https://www.deeplearningbook.org/")
return "\n".join(lines)
SAMPLE_GOAL = "I want to train convolutional models for image classification and debug them"
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description="Plan a prerequisite-closed reading path through the Deep Learning book.",
epilog="Exit codes: 0 ok · 3 goal outside the book · 4 goal too vague to route.",
)
parser.add_argument("--goal", help="what you want to be able to do afterwards")
parser.add_argument("--background", default="applied", choices=sorted(BACKGROUNDS),
help="your starting point (default: applied)")
parser.add_argument("--hours-per-week", type=float, default=5.0,
help="study hours available per week (default: 5)")
parser.add_argument("--include-intro", action="store_true",
help="add ch01 to the path; it is context rather than content, "
"so no lane pulls it in on its own (the complete read "
"already includes it)")
parser.add_argument("--output", choices=("text", "json"), default="text")
parser.add_argument("--sample", action="store_true",
help="run against a built-in example goal")
args = parser.parse_args(argv)
goal = SAMPLE_GOAL if args.sample else args.goal
if not goal:
parser.error("--goal is required (or use --sample)")
if args.hours_per_week <= 0:
parser.error("--hours-per-week must be positive")
scoped_out = out_of_scope_hits(goal)
lane_scores = score_lanes(goal)
top_score = lane_scores[0][1] if lane_scores else 0
# An out-of-scope subject wins over a weak lane match: "LoRA fine-tuning" hits the
# practitioner lane on the word "tuning" while being entirely outside the book.
if scoped_out and (len(scoped_out) >= 2 or top_score < 2):
payload = {
"status": "out_of_scope",
"goal": goal,
"reasons": scoped_out,
"pointer": "references/book_to_2026_delta.md",
}
if args.output == "json":
print(json.dumps(payload, indent=2))
else:
print("OUT OF SCOPE — the 2016 book does not cover this goal.\n")
for reason in scoped_out:
print(f" - {reason}")
print("\nSee references/book_to_2026_delta.md for what replaced the book's "
"position, and route to the skills named above.")
return 3
if not lane_scores:
payload = {
"status": "unroutable",
"goal": goal,
"questions": [
"What do you want to be able to DO afterwards — train, diagnose, "
"derive, or evaluate?",
"Which data type: images, sequences, tabular, or generative modelling?",
"Is this a refresher over known material or a first pass?",
],
"lanes": {key: lane["label"] for key, lane in LANES.items()},
}
if args.output == "json":
print(json.dumps(payload, indent=2))
else:
print("CANNOT ROUTE — the goal does not name a subject in the book.\n")
print("Answer one of these and re-run:")
for question in payload["questions"]:
print(f" - {question}")
print("\nOr name a lane directly:")
for key, label in payload["lanes"].items():
print(f" {key:<15} {label}")
return 4
result = plan(goal, args.background, args.hours_per_week, args.include_intro)
if scoped_out:
result["scope_warnings"] = scoped_out
if args.output == "json":
print(json.dumps(result, indent=2))
else:
print(render(result))
if scoped_out:
print("\nPartly outside the book — these parts are not covered:")
for reason in scoped_out:
print(f" - {reason}")
return 0
if __name__ == "__main__":
sys.exit(main())

306
engineering/deep-learning-book/skills/deep-learning-book/scripts/training_diagnostics.py Normal file
View file

@ -0,0 +1,306 @@
#!/usr/bin/env python3
"""training_diagnostics.py — Chapter 11's triage as an executable decision tree.
Chapter 11 argues that knowing many algorithms matters less than knowing which one a
measurement calls for, and gives the rule most teams get backwards: *read training
error first*. High training error means the model or the optimizer is the bottleneck,
and collecting more data will not help.
This tool takes the measurements you already have from a run and returns a ranked
diagnosis, the specific next action, and the chapter that justifies it. Rules fire in
priority order, so a NaN is never diagnosed as overfitting.
The thresholds are documented defaults, not physics — every one is overridable, and
--output json prints the values actually used.
Standard library only. No network calls, no model loading.
Exit codes:
0 a diagnosis was produced and training looks healthy
1 a diagnosis was produced and something needs fixing
2 usage error (argparse)
4 not enough measurements to diagnose — the missing ones are named
"""
from __future__ import annotations
import argparse
import json
import math
import sys
# Documented defaults. Every one is overridable from the command line.
DEFAULTS = {
# Relative gap = (val - train) / max(train, floor). Above this, overfitting.
"overfit_rel_gap": 0.30,
# Below this relative gap, the run is not overfitting in any actionable sense.
"healthy_rel_gap": 0.10,
# Gradient norm above this is a cliff (ch08); clip before anything else.
"grad_explode": 100.0,
# Gradient norm below this with non-trivial loss means saturation or dead units.
"grad_vanish": 1e-6,
# Divisor floor so a near-zero training loss does not produce an infinite ratio.
"loss_floor": 1e-6,
# Training loss above target * this multiple counts as underfitting.
"underfit_multiple": 1.15,
}
class Finding:
def __init__(self, rank: int, code: str, verdict: str, evidence: str,
action: str, chapter: str) -> None:
self.rank = rank
self.code = code
self.verdict = verdict
self.evidence = evidence
self.action = action
self.chapter = chapter
def as_dict(self) -> dict:
return {
"rank": self.rank,
"code": self.code,
"verdict": self.verdict,
"evidence": self.evidence,
"action": self.action,
"chapter": self.chapter,
}
def diagnose(train_loss: float | None, val_loss: float | None,
grad_norm: float | None, target_loss: float | None,
tiny_subset_fits: str, nan_seen: bool,
epochs: int | None, thresholds: dict) -> tuple[list[Finding], dict]:
"""Return (findings, metrics). Rules fire in priority order; rank 1 acts first."""
findings: list[Finding] = []
metrics: dict = {}
non_finite = (
nan_seen
or (train_loss is not None and not math.isfinite(train_loss))
or (val_loss is not None and not math.isfinite(val_loss))
)
# --- Rule 1: numerics before modelling (ch04) -------------------------------
if non_finite:
findings.append(Finding(
len(findings) + 1, "NUMERICS",
"Loss is NaN or infinite — this is a numerics failure, not a modelling one",
"a non-finite loss was reported",
"Check in this order: log(0) or log of a negative, division by a near-zero "
"denominator, exp of a large logit, then an exploding gradient. Use fused "
"log_softmax / cross-entropy rather than hand-rolled exp-then-normalize, and "
"work in log-space for probability products.",
"ch04 (Numerical Computation)",
))
# A non-finite loss makes every downstream ratio meaningless.
return findings, metrics
# --- Rule 2: is it a bug at all? (ch11) -------------------------------------
if tiny_subset_fits == "no":
findings.append(Finding(
len(findings) + 1, "BUG",
"The model cannot overfit a tiny subset — this is a bug, not a hard problem",
"--tiny-subset-fits no",
"Stop tuning. Check the data pipeline (labels aligned with inputs?), the loss "
"reduction, whether gradients actually reach the parameters (a detached tensor "
"or a frozen module), and the learning rate. Compare backprop against numerical "
"derivatives on one layer if it is still unclear.",
"ch11 (Practical Methodology — debugging strategies)",
))
# --- Rule 3: gradient behaviour (ch08, ch10, ch06) --------------------------
if grad_norm is not None:
metrics["grad_norm"] = grad_norm
if grad_norm > thresholds["grad_explode"]:
findings.append(Finding(
len(findings) + 1, "EXPLODING_GRADIENT",
"Gradient norm is in cliff territory",
f"grad_norm {grad_norm:g} > {thresholds['grad_explode']:g}",
"Clip gradients by global norm before changing anything else. If a "
"recurrence is in the graph, clipping is not optional. Then re-check the "
"learning rate and the initialization scale.",
"ch08 (Optimization — cliffs), ch10 (exploding gradients through time)",
))
elif grad_norm < thresholds["grad_vanish"]:
findings.append(Finding(
len(findings) + 1, "VANISHING_GRADIENT",
"Gradient norm is effectively zero — units are saturated or dead",
f"grad_norm {grad_norm:g} < {thresholds['grad_vanish']:g}",
"Check for saturating hidden units (sigmoid/tanh in a deep stack), dead "
"ReLUs from a too-large learning rate, and initialization scale "
"(He/Xavier). Add residual connections or normalization if the stack is "
"deep.",
"ch08 (initialization), ch06 (hidden units), ch10 (vanishing gradients)",
))
# --- Rule 4/5: the fit verdict (ch05, ch07, ch11) ---------------------------
if train_loss is not None and val_loss is not None:
denom = max(abs(train_loss), thresholds["loss_floor"])
gap = val_loss - train_loss
rel_gap = gap / denom
metrics.update({
"train_loss": train_loss,
"val_loss": val_loss,
"gap": round(gap, 6),
"relative_gap": round(rel_gap, 4),
})
underfitting = None
if target_loss is not None:
metrics["target_loss"] = target_loss
underfitting = train_loss > target_loss * thresholds["underfit_multiple"]
if underfitting:
findings.append(Finding(
len(findings) + 1, "UNDERFIT",
"Training error is above target — the bottleneck is capacity or optimization",
f"train_loss {train_loss:g} > target {target_loss:g} × "
f"{thresholds['underfit_multiple']}",
"Do NOT collect more data — it cannot help while training error is high. "
"In order: tune learning rate and schedule, check initialization, add "
"capacity, remove regularization you added earlier, train longer.",
"ch11 (the data-vs-capacity rule), ch08 (Optimization), ch05 (capacity)",
))
elif rel_gap > thresholds["overfit_rel_gap"]:
findings.append(Finding(
len(findings) + 1, "OVERFIT",
"Validation error substantially exceeds training error",
f"relative gap {rel_gap:.2f} > {thresholds['overfit_rel_gap']}",
"Work the regularization ladder in cost order, one change at a time: more "
"real data → label-preserving augmentation → early stopping → decoupled "
"weight decay (AdamW, not Adam+L2) → label smoothing → dropout. Shrink the "
"model LAST: past the interpolation threshold, double descent means a "
"bigger model with more data can generalize better.",
"ch07 (Regularization), ch05 (capacity and the U-curve's caveat)",
))
elif rel_gap < thresholds["healthy_rel_gap"] and target_loss is None:
findings.append(Finding(
len(findings) + 1, "GAP_SMALL_TARGET_UNKNOWN",
"The train/val gap is small — but without a target loss this cannot "
"distinguish 'converged' from 'underfitting equally on both splits'",
f"relative gap {rel_gap:.2f} < {thresholds['healthy_rel_gap']}, "
"no --target-loss supplied",
"Supply --target-loss (a human baseline, a published number, or the "
"irreducible-error estimate for the task) and re-run. Chapter 11's first "
"step is naming the metric and its target value, and this is why.",
"ch11 (determine goals: error metric and target value)",
))
if not findings:
findings.append(Finding(
1, "HEALTHY",
"No rule fired — the measurements supplied look healthy",
"; ".join(f"{k}={v}" for k, v in metrics.items()) or "measurements within thresholds",
"Confirm the metric you are optimizing is the one you care about, then change "
"one thing per experiment and keep the log.",
"ch11 (Practical Methodology)",
))
if epochs is not None:
metrics["epochs"] = epochs
return findings, metrics
def render(findings: list[Finding], metrics: dict, thresholds: dict) -> str:
lines = ["TRAINING DIAGNOSIS", "=" * 68]
if metrics:
lines.append("Measurements: " + " ".join(f"{k}={v}" for k, v in metrics.items()))
lines.append("")
for finding in findings:
lines.append(f"[{finding.rank}] {finding.code} — {finding.verdict}")
lines.append(f" evidence : {finding.evidence}")
lines.append(f" action : {finding.action}")
lines.append(f" chapter : {finding.chapter}")
lines.append("")
lines.append("Thresholds used: " + ", ".join(f"{k}={v}" for k, v in thresholds.items()))
lines.append("Rules fire in priority order — act on [1] before anything below it.")
return "\n".join(lines)
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(
description="Diagnose a training run against Chapter 11's decision tree.",
epilog="Exit codes: 0 healthy · 1 something needs fixing · 4 not enough input.",
)
parser.add_argument("--train-loss", type=float, help="final or current training loss")
parser.add_argument("--val-loss", type=float, help="matching validation loss")
parser.add_argument("--target-loss", type=float,
help="the loss value that would count as success — a human "
"baseline, a published number, or an irreducible-error "
"estimate. Without it, underfitting cannot be detected.")
parser.add_argument("--grad-norm", type=float, help="observed global gradient norm")
parser.add_argument("--epochs", type=int, help="epochs trained so far (recorded only)")
parser.add_argument("--nan", action="store_true",
help="the loss went NaN or inf at any point")
parser.add_argument("--tiny-subset-fits", choices=("yes", "no", "unknown"),
default="unknown",
help="can the model drive training loss to ~0 on 10-50 examples? "
"(ch11's smoke test; default: unknown)")
parser.add_argument("--overfit-rel-gap", type=float, default=DEFAULTS["overfit_rel_gap"])
parser.add_argument("--healthy-rel-gap", type=float, default=DEFAULTS["healthy_rel_gap"])
parser.add_argument("--grad-explode", type=float, default=DEFAULTS["grad_explode"])
parser.add_argument("--grad-vanish", type=float, default=DEFAULTS["grad_vanish"])
parser.add_argument("--underfit-multiple", type=float,
default=DEFAULTS["underfit_multiple"])
parser.add_argument("--output", choices=("text", "json"), default="text")
parser.add_argument("--sample", action="store_true",
help="run against a built-in overfitting example")
args = parser.parse_args(argv)
if args.sample:
args.train_loss, args.val_loss = 0.02, 1.90
args.grad_norm, args.epochs = 0.4, 30
args.tiny_subset_fits = "yes"
thresholds = {
"overfit_rel_gap": args.overfit_rel_gap,
"healthy_rel_gap": args.healthy_rel_gap,
"grad_explode": args.grad_explode,
"grad_vanish": args.grad_vanish,
"loss_floor": DEFAULTS["loss_floor"],
"underfit_multiple": args.underfit_multiple,
}
have_losses = args.train_loss is not None and args.val_loss is not None
if not have_losses and not args.nan and args.grad_norm is None \
and args.tiny_subset_fits == "unknown":
missing = {
"status": "insufficient_input",
"need_at_least_one_of": [
"--train-loss with --val-loss",
"--grad-norm",
"--nan",
"--tiny-subset-fits yes|no",
],
"note": "Chapter 11's first instruction is to instrument the run. This tool "
"reads instruments; it does not guess.",
}
if args.output == "json":
print(json.dumps(missing, indent=2))
else:
print("NOT ENOUGH INPUT — supply at least one of:")
for item in missing["need_at_least_one_of"]:
print(f" {item}")
print(f"\n{missing['note']}")
return 4
findings, metrics = diagnose(
args.train_loss, args.val_loss, args.grad_norm, args.target_loss,
args.tiny_subset_fits, args.nan, args.epochs, thresholds,
)
if args.output == "json":
print(json.dumps({
"findings": [f.as_dict() for f in findings],
"metrics": metrics,
"thresholds": thresholds,
}, indent=2))
else:
print(render(findings, metrics, thresholds))
return 0 if findings[0].code == "HEALTHY" else 1
if __name__ == "__main__":
sys.exit(main())

22
engineering/spinning-up-deep-rl/.claude-plugin/authoring-notes.json Normal file
View file

@ -0,0 +1,22 @@
{
"source": {
"spec": "generated by engineering/book-to-skill",
"build_pattern": "book-to-skill conversion: extract -> analyze -> chapter files + glossary + patterns + cheatsheet -> master SKILL.md",
"distinct_from": "not a hand-authored skill; every claim traces to the source document named below",
"source_document": "Spinning Up in Deep RL by Joshua Achiam (OpenAI)",
"chapters": 20,
"distribution": "shareable",
"license_scope": "plugin.json's top-level `license` covers this package's scaffolding only. The compiled notes under skills/ are derived from the source document and carry that work's terms; see source.rights_basis.",
"rights_basis": "open-license",
"rights_note": "the source work carries a licence permitting derivative distribution"
},
"attribution": {
"derived_from": "https://github.com/openai/spinningup",
"upstream_docs": "https://spinningup.openai.com/",
"upstream_path": "docs/ (reStructuredText tree, 38 files)",
"original_author": "Joshua Achiam, OpenAI",
"original_license": "MIT",
"original_copyright": "Copyright (c) 2018 OpenAI (http://openai.com)",
"derivation_note": "No upstream code or prose is reproduced. The compiled skill is structured study notes -- named frameworks, key concepts, decision rules and per-chapter summaries -- generated by engineering/book-to-skill from the source's docs/ tree, with chapter structure taken from the source's own toctree. Upstream's MIT notice is reproduced in full in this package's LICENSE, which is where the licence obligation is met; this file is authoring metadata and is not a licence notice. Coverage is pinned to the source's final (January 2020 PyTorch) update."
}
}

15
engineering/spinning-up-deep-rl/.claude-plugin/plugin.json Normal file
View file

@ -0,0 +1,15 @@
{
"name": "spinning-up-deep-rl",
"description": "Knowledge base from \"Spinning Up in Deep RL\" by Joshua Achiam (OpenAI, MIT-licensed). Use when applying Achiam's frameworks for RL fundamentals and MDPs, the model-free algorithm taxonomy, policy gradient derivations, the six reference algorithms (VPG, TRPO, PPO, DDPG, TD3, SAC), debugging silently-failing RL code, or running rigorous multi-seed RL experiments.",
"version": "1.0.0",
"author": {
"name": "Alireza Rezvani",
"url": "https://github.com/alirezarezvani"
},
"homepage": "https://github.com/alirezarezvani/claude-skills/tree/main/engineering/spinning-up-deep-rl",
"repository": "https://github.com/alirezarezvani/claude-skills",
"license": "MIT",
"skills": [
"./skills/spinning-up-deep-rl"
]
}

58
engineering/spinning-up-deep-rl/LICENSE Normal file
View file

@ -0,0 +1,58 @@
This package has two layers with two different terms.
------------------------------------------------------------------------------
1. Package scaffolding — plugin manifest, agent, command, README
------------------------------------------------------------------------------
MIT License
Copyright (c) 2026 Alireza Rezvani
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
------------------------------------------------------------------------------
2. Compiled notes under skills/spinning-up-deep-rl/
------------------------------------------------------------------------------
Derived from OpenAI's "Spinning Up in Deep RL" documentation
(https://spinningup.openai.com/ · https://github.com/openai/spinningup),
primarily developed by Joshua Achiam. Distributed under the same MIT terms,
whose notice is reproduced below in full as that licence requires.
The MIT License
Copyright (c) 2018 OpenAI (http://openai.com)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.

49
engineering/spinning-up-deep-rl/README.md Normal file
View file

@ -0,0 +1,49 @@
# Spinning Up in Deep RL
Knowledge-base plugin compiled from **Spinning Up in Deep RL by Joshua Achiam (OpenAI)** by
[`engineering/book-to-skill`](../book-to-skill/). 20 chapters indexed.
## What is in here
| File | Contents |
|------|----------|
| `skills/spinning-up-deep-rl/SKILL.md` | Core frameworks, chapter index, topic index (resident, under 4k tokens) |
| `skills/spinning-up-deep-rl/chapters/` | One summary per chapter — loaded on demand, never all at once |
| `skills/spinning-up-deep-rl/glossary.md` | Every significant term, alphabetized, with its chapter |
| `skills/spinning-up-deep-rl/patterns.md` | Techniques and design patterns with trade-offs |
| `skills/spinning-up-deep-rl/cheatsheet.md` | Decision rules, thresholds and trade-off matrices |
## Use
```
/cs:spinning-up-deep-rl # core frameworks + chapter index
/cs:spinning-up-deep-rl <topic> # resolve via topic index, read one chapter
/cs:spinning-up-deep-rl ch05 # read one chapter summary
```
Or invoke the `cs-spinning-up-deep-rl` agent for a working session anchored to this source.
## Provenance and limits
**Source:** OpenAI's [Spinning Up in Deep RL](https://spinningup.openai.com/)
([openai/spinningup](https://github.com/openai/spinningup)), primarily developed by
**Joshua Achiam**. Compiled from the `docs/` reStructuredText tree at the January 2020
PyTorch update.
**Rights basis:** `open-license`. The source is **MIT, Copyright (c) 2018 OpenAI**, which
permits derivative distribution. The full upstream notice is reproduced in
[`LICENSE`](LICENSE) alongside this package's own; the top-level `license` field in
`plugin.json` covers the scaffolding only.
Generated, not hand-authored: every claim traces to the source document. It carries that source's
blind spots, and it is a set of structured notes — **not a copy of the work and not a substitute
for reading it**.
**What it does not cover:** DQN and the discrete-action value-learning family, recurrent or
convolutional architectures, partially-observed settings, model-based implementations, and any
deep RL work after early 2020. The six implementations documented are educational; `ch13` records
which are research-grade (DDPG, TD3, SAC) and which are not (VPG, TRPO, PPO).
Distribution: `shareable`. Regenerate or extend with
`python3 engineering/book-to-skill/skills/book-to-skill/scripts/extract_document.py`, then re-run
`book_skill_validator.py` before loading the result.

37
engineering/spinning-up-deep-rl/agents/cs-spinning-up-deep-rl.md Normal file
View file

@ -0,0 +1,37 @@
---
name: cs-spinning-up-deep-rl
description: Answers from the knowledge base compiled from Spinning Up in Deep RL by Joshua Achiam (OpenAI). Loads the master frameworks first and reads a single chapter file on demand rather than the whole source. Refuses to answer beyond what the source covers.
skills: engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl
domain: engineering
model: opus
tools: [Read, Grep, Glob]
---
# Spinning Up in Deep RL — Knowledge Agent
## Voice
**Opening:** "Which framework or chapter are you reaching for?"
**Forcing question:** "Is this something the source actually covers, or are you asking me to
extrapolate past it?"
**Closing:** "That is the author's formulation, from ch<N>. Anything past it is my inference, not theirs."
## Purpose
Applies the frameworks compiled from **Spinning Up in Deep RL by Joshua Achiam (OpenAI)** (20 chapters
indexed) while the user works. Answers with the author's exact naming, then cites the chapter.
## How it navigates
1. Read `skills/spinning-up-deep-rl/SKILL.md` — Core Frameworks and both indexes.
2. Match the question against the Topic Index; read **only** the chapter files it points to.
3. Reach for `glossary.md` for a term, `patterns.md` for a technique, `cheatsheet.md` for a decision.
4. Never load every chapter — that is the cost this skill exists to avoid.
## Hard rules
- **Cite the chapter.** Every framework claim names the chapter it came from.
- **Do not extrapolate silently.** If the source does not cover it, say so before answering from
general knowledge, and label which is which.
- **Preserve exact naming.** The author's term is the interface; a paraphrase breaks lookup.
- **Do not reproduce the source at length.** These are structured notes, not a copy of the work.

30
engineering/spinning-up-deep-rl/commands/cs-spinning-up-deep-rl.md Normal file
View file

@ -0,0 +1,30 @@
---
name: "cs-spinning-up-deep-rl"
description: "/cs:spinning-up-deep-rl [topic | framework | chNN] — query the knowledge base compiled from Spinning Up in Deep RL by Joshua Achiam (OpenAI). Use when applying its frameworks while working, looking up a term, or reading one chapter's summary."
---
# /cs:spinning-up-deep-rl — Spinning Up in Deep RL
**Command:** `/cs:spinning-up-deep-rl [topic | framework name | chNN]`
## When to run
- Applying a framework from this source to work in progress
- Looking up the author's exact formulation of a term
- Reading one chapter's compiled summary without opening the source
- Checking whether the source covers a question at all
## What it does
1. Loads `engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/SKILL.md` — Core Frameworks plus the Chapter and Topic indexes.
2. **No argument** → reports the core frameworks and the chapter index.
3. **A topic or framework name** → resolves it through the Topic Index and reads only the
matching chapter file.
4. **`chNN`** → reads that chapter's summary directly.
5. Answers with the author's naming and cites the chapter.
## Boundary
This command answers from **one source** (20 chapters indexed). Anything it does not
cover gets said out loud rather than filled in — and hands-on work in your codebase belongs to the
engineering skills, not here.

192
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/SKILL.md Normal file
View file

@ -0,0 +1,192 @@
---
name: spinning-up-deep-rl
description: "Knowledge base from \"Spinning Up in Deep RL\" by Joshua Achiam (OpenAI, MIT-licensed). Use when applying Achiam's frameworks for RL fundamentals and MDPs, the model-free algorithm taxonomy, policy gradient derivations, the six reference algorithms (VPG, TRPO, PPO, DDPG, TD3, SAC), debugging silently-failing RL code, or running rigorous multi-seed RL experiments."
---
# Spinning Up in Deep RL
**Author**: Joshua Achiam (OpenAI) | **Source**: spinningup.readthedocs.io, MIT | **Chapters**: 20 | **Generated**: 2026-08-25
## How to Use This Skill
- **No argument** — load the core frameworks below
- **A topic** — ask about `advantage function`, `target networks`, `entropy regularization`;
I resolve it through the Topic Index and read that chapter file
- **`chNN`** — I load that chapter's summary
- **"what chapters do you have?"** — the full index
```
/cs:spinning-up-deep-rl # core frameworks + chapter index
/cs:spinning-up-deep-rl entropy regularization # topic index -> ch19, read that chapter
/cs:spinning-up-deep-rl ch09 # one chapter summary
```
When you ask about something not in Core Frameworks, I read the relevant chapter file before
answering rather than guessing from the index.
---
## Core Frameworks & Mental Models
### The RL problem
`pi* = argmax_pi J(pi)`, where `J(pi) = E_{tau~pi}[R(tau)]`. Every algorithm approximates
this; where it substitutes a different objective (a Bellman residual, a surrogate), that
substitution is the source of its failure modes. Four value functions — `V^pi`, `Q^pi`, `V*`,
`Q*` — all obey Bellman self-consistency, and `a*(s) = argmax_a Q*(s,a)` is why Q-learning
is a viable family at all. Advantage `A^pi(s,a) = Q^pi(s,a) - V^pi(s)` is the relative-quality
signal policy gradients run on. (ch07)
### The two branching questions
Place any algorithm by asking: **does it have or learn a model**, and **what does it learn**
(policy, Q-function, value function, model). That generates the whole landscape. (ch08)
### Policy optimization vs Q-learning — the central trade-off
- **Policy optimization** is *principled: you directly optimize the thing you want.* Stable
and reliable. On-policy, so it cannot reuse data, so it is sample-hungry.
- **Q-learning** only *indirectly* optimizes performance, by training `Q_theta` to satisfy a
self-consistency equation. Many failure modes, so less stable. But substantially more
sample efficient when it works, because it reuses everything.
- **Satisfying the Bellman equations well carries no guarantee of good policy performance.**
- The two are not exclusive — DDPG and SAC live between them deliberately. (ch08)
### The policy gradient template
`grad J = E[ sum_t grad log pi_theta(a_t|s_t) * Phi_t ]`. Five valid choices of `Phi_t`:
full return, reward-to-go, reward-to-go minus a baseline, `Q^pi`, and `A^pi`. All share an
expectation and differ in variance. Two rules get you from the first to the last:
- **Don't let the past distract you** — drop rewards obtained before the action. Those terms
had zero mean and nonzero variance: pure noise.
- **Baselines** — by the EGLP lemma, any state-only `b(s)` can be added or subtracted freely.
The standard choice is `V^pi(s_t)`, learned by MSE regression onto reward-to-go. (ch09)
### The policy-gradient loss is not a loss function
Its data distribution depends on the parameters, and it does not measure performance even in
expectation. Only at the current parameters, with data from those parameters, does it have
the negative gradient of performance. **You can send it to negative infinity while performance
craters, and it usually will. Only average return means anything.** (ch09)
### Broken RL code almost always fails silently
It runs fine; the agent just never learns. Usually something is computed with the wrong
equation, on the wrong distribution, or piped to the wrong place. **If it doesn't work, assume
there's a bug** before touching hyperparameters. Debug by measuring everything and reading the
code critically. The archetype is one missing `squeeze`: a `[N]` vs `[N,1]` shape mismatch is
broadcast-compatible, raises nothing, and silently turns the Bellman backup into an `[N,N]`
matrix. (ch10, ch12)
### Learn by doing
Write your own implementations, **shortest correct version** of each, **simplest algorithms
first**. VPG, DQN, A2C, PPO, DDPG, roughly in that order; ~250-300 lines each. Single-threaded
before parallel. **Iterate fast in simple environments — under 5 minutes turnaround at the
debug stage.** Do not attempt Atari or Humanoid before the toy task works. Read papers for
their ablations and supplementary material, but **do not overfit to paper details** (the
original DDPG's architecture, init scheme and batch norm are not strictly necessary) **or to
existing implementations** (their abstractions serve reuse, not your single use case). (ch10)
### Rigor: four standards
1. **Fair comparisons** — tune the baseline as hard as your method; never handicap it.
2. **Remove stochasticity as a confounder** — at least 3 seeds, 10 or more to be thorough.
Two seed groups can produce curves that look like different distributions.
3. **High-integrity experiments** — launch fresh final runs and precommit to reporting them.
Tuning produces hypotheses; final runs produce conclusions.
4. **Check each claim separately** — ablate every design decision. (ch10)
### The safe-step family (on-policy)
VPG takes an unconstrained gradient step, so a single bad step can collapse performance.
**TRPO** constrains the step in **KL-divergence between policies, not distance in parameter
space**, then backtracking-line-searches until the exact constraint holds. **PPO** drops the
constraint and instead **clips the objective so the policy gains nothing by moving far**,
which is first-order, far simpler, and empirically at least as good. (ch14, ch15, ch16)
### The overestimation family (off-policy)
DDPG amortizes the intractable continuous `max_a Q(s,a)` into a learned policy:
`max_a Q(s,a) ~= Q(s, mu(s))`. It needs a **replay buffer** (licensed because the Bellman
equation is indifferent to how data was collected) and **target networks** (because the target
otherwise depends on the parameters being trained). Its failure mode is **Q-value
overestimation, which the policy actively exploits**. **TD3** answers with clipped double-Q,
delayed policy updates and target policy smoothing. **SAC** adds **entropy regularization**,
making the explore-exploit trade-off an explicit coefficient `alpha`. (ch17, ch18, ch19)
---
## Chapter Index
| # | Title | Key Frameworks |
|---|-------|----------------|
| [ch01](chapters/ch01-introduction.md) | Introduction | The missing middle step, Code Design Philosophy |
| [ch02](chapters/ch02-installation.md) | Installation | Install-then-verify, MuJoCo optionality |
| [ch03](chapters/ch03-algorithm-lineup.md) | Algorithms: What's Included and Why | The two lineages, on/off-policy trade-off, code template |
| [ch04](chapters/ch04-running-experiments.md) | Running Experiments | One flag per kwarg, ExperimentGrid, save-dir suffixes |
| [ch05](chapters/ch05-experiment-outputs.md) | Experiment Outputs | Tools not files, watch-then-measure |
| [ch06](chapters/ch06-plotting-results.md) | Plotting Results | `Performance` alias, prefix autocompletion, seed averaging |
| [ch07](chapters/ch07-key-concepts-in-rl.md) | Part 1: Key Concepts in RL | MDPs, four value functions, Bellman equations, advantage |
| [ch08](chapters/ch08-kinds-of-rl-algorithms.md) | Part 2: Kinds of RL Algorithms | Taxonomy, model bias, policy-opt vs Q-learning |
| [ch09](chapters/ch09-intro-to-policy-optimization.md) | Part 3: Intro to Policy Optimization | Log-derivative trick, EGLP lemma, reward-to-go, baselines |
| [ch10](chapters/ch10-spinning-up-as-a-researcher.md) | Spinning Up as a Deep RL Researcher | Learn by doing, three idea frames, four rigor standards |
| [ch11](chapters/ch11-key-papers-in-deep-rl.md) | Key Papers in Deep RL | 13-section topic map |
| [ch12](chapters/ch12-exercises.md) | Exercises | Problem Set 1 and 2, the silent DDPG bug |
| [ch13](chapters/ch13-benchmarks.md) | Benchmarks | The parity disclosure, family-specific metrics |
| [ch14](chapters/ch14-vpg.md) | Vanilla Policy Gradient | The six-step loop |
| [ch15](chapters/ch15-trpo.md) | Trust Region Policy Optimization | KL trust region, line search, conjugate gradient |
| [ch16](chapters/ch16-ppo.md) | Proximal Policy Optimization | PPO-Clip, KL early stopping |
| [ch17](chapters/ch17-ddpg.md) | Deep Deterministic Policy Gradient | MSBE, replay buffers, target networks, polyak |
| [ch18](chapters/ch18-td3.md) | Twin Delayed DDPG | Clipped double-Q, delayed updates, target smoothing |
| [ch19](chapters/ch19-sac.md) | Soft Actor-Critic | Entropy regularization, reparameterization, squashed Gaussian |
| [ch20](chapters/ch20-logger-and-utilities.md) | Logger, MPI Tools and Run Utils | EpochLogger pattern, MPI PyTorch order |
## Topic Index
- **Advantage function** ch07, ch09, ch14
- **Baselines** ch09
- **Bellman equations** ch07, ch17
- **Benchmarks / parity** ch13, ch01
- **Clipped double-Q** ch18, ch19
- **Continuous action spaces** ch07, ch17
- **Debugging / silent failure** ch10, ch12
- **DDPG** ch17, ch03, ch08
- **Entropy regularization** ch19
- **Exploration vs exploitation** ch14, ch17, ch19
- **GAE** ch09, ch14
- **Installation** ch02
- **KL divergence / trust region** ch15, ch16
- **Logging** ch20, ch05
- **MDPs** ch07
- **Model-based RL** ch08
- **MPI / parallelization** ch20, ch02, ch04
- **MSBE** ch17
- **Off-policy** ch03, ch08, ch17
- **On-policy** ch03, ch08, ch14
- **Papers / literature** ch11, ch10
- **Plotting** ch06, ch13
- **Policies (categorical, Gaussian, squashed)** ch07, ch19
- **Policy gradient derivation** ch09
- **PPO** ch16, ch03
- **Q-learning** ch08, ch07
- **Replay buffer** ch17
- **Reparameterization trick** ch19, ch10
- **Research process / rigor** ch10, ch13
- **Reward-to-go** ch09
- **Running experiments** ch04, ch05
- **SAC** ch19, ch03, ch08
- **Seeds / variance** ch10, ch13, ch04
- **Target networks / polyak** ch17, ch18
- **TD3** ch18, ch12
- **TRPO** ch15, ch03
- **Value functions** ch07, ch09
- **VPG** ch14, ch09
## Supporting Files
- [glossary.md](glossary.md) · [patterns.md](patterns.md) · [cheatsheet.md](cheatsheet.md)
## Scope & Limits
Covers the Spinning Up documentation only, as of the January 2020 PyTorch update. It does
**not** cover: DQN and the discrete-action value-learning family (referenced, never
implemented here), recurrent or convolutional architectures, partially-observed settings,
model-based implementations, exploration/meta-RL/hierarchy beyond ch11's reading list, or any
deep RL work after early 2020. The six implementations are educational; ch13 says which are
research-grade. For topics beyond this source, I say so rather than improvising.
---
*Compiled from OpenAI's Spinning Up in Deep RL documentation (MIT, Copyright (c) 2018 OpenAI),
primarily developed by Joshua Achiam. Structured study notes, not a reproduction of the source.*

102
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch01-introduction.md Normal file
View file

@ -0,0 +1,102 @@
# Chapter 1: Introduction
## Core Idea
Spinning Up exists to be the missing middle step between "I want to work on deep RL"
and "I can implement and evaluate a deep RL algorithm" — the field has no standard
textbook, papers omit the design details that decide whether code works, and production
RL libraries hide the algorithm inside framework abstractions.
## Frameworks Introduced
- **The missing middle step**: the gap Spinning Up fills, between high-level awareness
(what topics exist, why they matter) and the ability to transmute an algorithm into code.
- When to use: deciding whether a resource teaches you deep RL or merely describes it.
- How: judge a resource on whether it closes the theory-to-code gap. Papers give theory
and omit details; libraries give code that hides the algorithm. Neither closes it alone.
- **Code Design Philosophy** (the four rules the implementations obey):
- **As simple as possible while still being reasonably good.** Not state of the art;
good enough to reach roughly the intended performance.
- **Highly consistent with each other**, so understanding one makes the next painless.
- **Almost completely self-contained** — virtually no shared code between algorithms
except logging, saving, loading and MPI utilities. You can study one algorithm without
following a chain of dependencies.
- **Patterned to come as close to pseudocode as possible**, minimizing the theory-code gap.
- How: when reading or writing RL code, prefer the version you can read top to bottom
over the version that reuses the most code. Reuse is an engineering virtue that is a
pedagogical cost.
- **Minimize tricks and minimize differences**: the implementations deliberately omit
tricks present in the original papers (e.g. the regularization terms in the original
Soft Actor-Critic code, observation normalization in all algorithms) and deliberately
remove gratuitous differences between similar algorithms (DDPG, TD3 and SAC all run
gradient updates after fixed intervals of environment interaction, so they compare cleanly).
- When to use: any time you are comparing two algorithms and want the comparison to be
about the algorithms rather than about their implementation choices.
## Key Concepts
- **Deep RL**: reinforcement learning (learning to solve tasks by trial and error) combined
with deep learning.
- **Reasonably good**: achieves roughly the intended performance but does not necessarily
match the best reported results in the literature on every task.
- **Maintenance mode**: Spinning Up's current status. Breaking bugs get repaired; no major
new features are planned.
- **The Rosetta Stone goal**: the motivation for the January 2020 PyTorch update — the same
algorithm expressed in two neural network libraries so the algorithm is separable from the
library.
## Mental Models
- Think of an RL library's abstraction layer as a **tax on learning**: good for code reuse
between algorithms, unnecessary if you are writing one algorithm for one use case.
- Use "**can I read this algorithm without opening another file?**" as the test of whether
an implementation is written to be learned from.
- Think of the resource landscape as **two failure modes**: papers that obscure key design
details, and public implementations that are hard to read. Spinning Up targets both.
## Anti-patterns
- **Using Spinning Up's implementations for scientific benchmarking comparisons**: they are
"reasonably good," not best-reported. See ch13 for which ones are at parity (DDPG, TD3, SAC)
and which are not (VPG, TRPO, PPO). Use OpenAI Baselines for TRPO/PPO research comparisons.
- **Assuming an educational implementation is a research implementation**: the omitted tricks
(observation normalization, normalized value regression targets) are exactly what separates
the two.
- **Treating deep RL as engineering-only**: the material explicitly serves people from
professions with no connection to engineering or computer science who nonetheless need to
make informed decisions about the technology.
## Worked Example
Support history, read as a record of what the community actually asked for after release:
| Date | Event |
|------|-------|
| Nov 8, 2018 | Initial release, followed by three weeks of high-bandwidth support |
| April 2019 | Six-month internal review of community feedback |
| Jan 2020 | The PyTorch update ships |
| Future | No major updates planned |
The April 2019 review surfaced exactly three requests, in priority order:
1. **Implementations in other neural network libraries** — enough people had written their
own PyTorch ports (Fired Up, Spinning Up Basic, Torching Up) that a "Rosetta Stone for
deep RL" became the top priority. This one shipped.
2. **Open source RL environments** (e.g. PyBullet) for benchmarks, examples and exercises,
to avoid the proprietary MuJoCo dependency. Did not ship.
3. **More algorithms**, especially Deep Q-Networks. Did not ship.
The lesson for anyone building an educational resource: the community's top request was not
more content, it was the same content expressed in the framework they already use.
## Key Takeaways
1. The barrier to entry in deep RL is not intelligence or math; it is that theory and code
are documented in separate places and neither is written to connect to the other.
2. Self-contained beats DRY when the reader is trying to learn the algorithm.
3. Consistency across implementations is a teaching feature: differences that remain are
real algorithmic differences.
4. "Reasonably good" is a deliberate, stated performance target — check ch13 before citing
any Spinning Up number as a benchmark.
5. Deep RL is central to AI safety work in OpenAI's framing; the resource exists partly as
a recruiting and capability-building pipeline for that.
## Connects To
- **Ch 3**: which algorithms were chosen and why, and the code template they all share.
- **Ch 10**: the essay on becoming a researcher — the practice counterpart to this chapter's
philosophy.
- **Ch 13**: the benchmark numbers that qualify "reasonably good."

88
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch02-installation.md Normal file
View file

@ -0,0 +1,88 @@
# Chapter 2: Installation
## Core Idea
Spinning Up needs Python 3, OpenAI Gym and OpenMPI, on Linux or OSX; MuJoCo is optional
but is the de facto benchmarking standard, and everything in the book works without it on
the free Classic Control and Box2D environments.
## Frameworks Introduced
- **Install then immediately verify with a real training run**: the install is not confirmed
by a successful `pip install`, it is confirmed by an agent visibly learning.
- When to use: every fresh environment, before debugging anything else.
- How: run the install test below, then watch the policy and plot the curve. Three
commands, three different subsystems (training, rendering, plotting) verified.
## Key Concepts
- **OpenMPI**: the message-passing library used for the parallelized (on-policy) algorithms.
Installed with `apt-get install libopenmpi-dev` on Ubuntu, `brew install openmpi` on Mac.
- **MuJoCo**: a proprietary physics engine; free to trial and free for full-time students,
otherwise paid. The de facto standard for benchmarking continuous-control deep RL.
- **mujoco-py**: the Python bindings that let Gym use MuJoCo.
- **Classic Control / Box2D**: Gym environment families that are completely free and
sufficient to get started.
- **Editable install**: `pip install -e .` from the cloned repo, so edits to the algorithm
files take effect without reinstalling — the intended workflow for a repo you are meant
to read and modify.
## Code Examples
Environment and dependencies:
```bash
conda create -n spinningup python=3.6
conda activate spinningup
# Ubuntu
sudo apt-get update && sudo apt-get install libopenmpi-dev
# Mac OS X (requires Homebrew)
brew install openmpi
git clone https://github.com/openai/spinningup.git
cd spinningup
pip install -e .
```
Verify the install (roughly 10 minutes; leave it running and keep reading):
```bash
python -m spinup.run ppo --hid "[32,32]" --env LunarLander-v2 \
--exp_name installtest --gamma 0.999
python -m spinup.run test_policy data/installtest/installtest_s0
python -m spinup.run plot data/installtest/installtest_s0
```
Optional MuJoCo, after following the mujoco-py README and obtaining a license:
```bash
pip install gym[mujoco,robotics]
python -m spinup.run ppo --hid "[32,32]" --env Walker2d-v2 --exp_name mujocotest
```
## Reference Tables
| Requirement | Status | Notes |
|-------------|--------|-------|
| Python 3 | Required | Anaconda recommended; the docs pin 3.6 |
| OpenAI Gym | Required | Installed by `pip install -e .`, MuJoCo envs excluded |
| OpenMPI | Required | Needed for the MPI-parallelized on-policy algorithms |
| Linux / OSX | Required | Windows unsupported and untested; one community workaround exists |
| MuJoCo | Optional | Proprietary, licensed; preferred because of benchmarking convention |
## Anti-patterns
- **Skipping the install test** and then debugging an algorithm that was never installed
correctly. The install test costs ten unattended minutes.
- **Treating a MuJoCo licence as a prerequisite for learning RL**: Classic Control and Box2D
are free and adequate for everything up to benchmarking.
- **Fighting package management ad hoc**: the docs go out of their way to point at conda
explainers, because "I just installed this thing but it says it's not found" is the
predicted failure, not an unusual one.
## Key Takeaways
1. Install is Python 3 + Gym + OpenMPI, plus optional MuJoCo; Linux or OSX only.
2. `pip install -e .` (editable) because you are expected to modify the code.
3. Verification means an actual short PPO run, a rendered policy and a plot.
4. Not having MuJoCo blocks benchmarking, not learning.
## Connects To
- **Ch 4**: `python -m spinup.run` is the entry point every later chapter uses.
- **Ch 13**: the benchmarks that make MuJoCo the convention.
- **Ch 20**: the MPI utilities that OpenMPI enables.

101
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch03-algorithm-lineup.md Normal file
View file

@ -0,0 +1,101 @@
# Chapter 3: Algorithms — What's Included and Why
## Core Idea
The six algorithms (VPG, TRPO, PPO, DDPG, TD3, SAC) are not a survey; they are two
lineages of ideas, each starting from a foundational algorithm and progressively fixing
its defining weakness, culminating in PPO and SAC.
## Frameworks Introduced
- **The two lineages**:
- **On-policy line: VPG to TRPO to PPO.** Each step buys back sample efficiency without
giving up the stability that comes from directly optimizing the objective you care about.
- **Off-policy line: DDPG to TD3 and SAC.** Each step mitigates the brittleness that comes
from optimizing a Bellman self-consistency condition instead of performance itself.
- When to use: placing any new algorithm you read about. Ask which line it is on and which
weakness of its predecessor it claims to fix.
- **The central trade-off the lineup exposes**: on-policy algorithms *directly optimize the
objective you care about* — policy performance — and it works out mathematically that
you need on-policy data to compute the updates, so they cannot reuse old data. Off-policy
algorithms exploit the Bellman optimality equations, which hold for *any* transition data,
so they reuse everything — but satisfying Bellman's equations well carries **no guarantee**
of good policy performance.
- How: choose the family by which risk you can afford. Stability with a sample budget, or
sample efficiency with a tuning budget.
- **The standard two-file code template**: every implementation splits into an *algorithm
file* (experience buffer class, then one function that runs the algorithm) and a *core
file* (utilities, the actor-critic constructors, the MLP actor-critic).
- When to use: reading any Spinning Up algorithm, or structuring your own.
## Key Concepts
- **On-policy**: each update uses only data collected by the most recent version of the policy.
- **Off-policy**: each update can use data collected at any point in training, however the
agent was exploring at the time.
- **MLP actor-critic**: all six use non-recurrent multi-layer-perceptron actor-critics, which
makes them suitable for fully-observed, non-image-based environments (e.g. Gym MuJoCo) and
unsuitable, as shipped, for partial observability or pixels.
- **Sample efficiency**: how much environment interaction is needed to reach a performance level.
- **Deterministic policy gradients**: the theory (2014) that made DDPG possible — much younger
than the policy gradient theory behind VPG, whose core elements go back to the late 80s.
## Reference Tables
| Algorithm | Family | Policy | Action spaces | Parallel (MPI) | PyTorch | TF1 |
|-----------|--------|--------|---------------|----------------|---------|-----|
| VPG | On-policy | Stochastic | Discrete + continuous | Yes | Yes | Yes |
| TRPO | On-policy | Stochastic | Discrete + continuous | Yes | No | Yes |
| PPO | On-policy | Stochastic | Discrete + continuous | Yes | Yes | Yes |
| DDPG | Off-policy | Deterministic | Continuous only | No | Yes | Yes |
| TD3 | Off-policy | Deterministic | Continuous only | No | Yes | Yes |
| SAC | Off-policy | Stochastic | Continuous only (as shipped) | No | Yes | Yes |
## Code Examples
The PyTorch algorithm function, in order — the template every implementation follows:
```
1) Logger setup
2) Random seed setting
3) Environment instantiation
4) Build the actor-critic module via the `actor_critic` function passed in as an argument
5) Instantiate the experience buffer
6) Set up callable loss functions that also return algorithm-specific diagnostics
7) Make PyTorch optimizers
8) Set up model saving through the logger
9) Set up an update function: one epoch of optimization, or one step of descent
10) Main loop: a) run the agent in the environment
b) periodically update parameters per the algorithm's main equations
c) log key performance metrics and save the agent
```
The TF1 version is the same shape with four graph-construction steps inserted (placeholders,
actor-critic graph, loss/diagnostic graph, training ops) and a session step.
## Mental Models
- Read the lineup as **"progressions of ideas from the recent history of the field"**, not as
a menu. VPG predates deep RL entirely; DDPG's theory is from 2014.
- Treat **PPO and SAC as the two defaults** — they are close to state of the art on
reliability and sample efficiency among policy-learning algorithms.
- Think of the `actor_critic` argument as the seam: the algorithm function is fixed, the
network construction is injected. That is also where the ch12 silent bug lives.
## Anti-patterns
- **Reaching for DDPG on a discrete action space.** DDPG, TD3 and the shipped SAC are
continuous-only. The max over actions is why (see ch17).
- **Expecting these to work on Atari or partially-observed tasks as shipped**: MLP,
non-recurrent, non-image.
- **Picking an algorithm by recency**: TD3 and SAC were published roughly concurrently; SAC
is not a successor to TD3, it is a parallel branch that borrowed the clipped double-Q trick.
## Key Takeaways
1. Two families, one trade-off: directly optimize performance (on-policy, stable, sample-hungry)
or exploit Bellman (off-policy, sample-efficient, no performance guarantee).
2. PPO and SAC are the practical end points of each line.
3. Continuous-only for the whole off-policy line, as implemented here.
4. Every implementation is two files and the same ten-step function; learn the template once.
5. TRPO has no PyTorch implementation in Spinning Up.
## Connects To
- **Ch 8**: the full taxonomy this lineup is a slice of.
- **Ch 14-19**: one chapter per algorithm, in lineage order.
- **Ch 12**: the exercises that make the on-policy/off-policy distinction concrete.

133
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch04-running-experiments.md Normal file
View file

@ -0,0 +1,133 @@
# Chapter 4: Running Experiments
## Core Idea
`spinup/run.py` turns every keyword argument of every algorithm into a command-line flag,
and turns a list of values for a flag into a series of experiments — so hyperparameter
sweeps and multi-seed runs are one command, not a script.
## Frameworks Introduced
- **One flag per kwarg**: if `kwarg` is a valid keyword argument of an algorithm function,
`--kwarg` sets it. `python -m spinup.run [algo] --help` prints the docstring readout.
- How: `python -m spinup.run [algo name] [experiment flags]`.
- **Multiple values means multiple experiments**: providing more than one value for an
argument launches one experiment per combination, **in series**. They do not run in
parallel because a single experiment already soaks up enough resources that concurrency
buys no speedup.
- When to use: seed sweeps (the ch10 rigor requirement) and hyperparameter ablations.
- **ExperimentGrid**: the in-script equivalent, based on but simpler than rllab's
VariantGenerator. `spinup.run` uses one under the hood.
- How: `eg.add(param_name, values, shorthand, in_name)` then `eg.run(thunk, **run_kwargs)`.
`in_name` forces a parameter into the experiment name even when it does not vary.
- Difference from the CLI: no shortcut kwargs — you must write `ac_kwargs:hidden_sizes`,
not `hid`.
- **Shorthand-driven save directories**: results land in
`data_dir/[outer_prefix]exp_name[suffix]/[inner_prefix]exp_name[suffix]_s[seed]`.
The suffix encodes only the hyperparameters that *differ across the launched experiments*,
and never the seed — so runs that differ only by seed group into the same folder, which is
exactly what the plotter needs to average over seeds.
## Key Concepts
- **`--env` / `--env_name`**: a Gym environment name, converted internally into the `env_fn`
callable every algorithm actually takes.
- **`--hid` / `--act`**: shortcut flags for `ac_kwargs:hidden_sizes` and `ac_kwargs:activation`,
valid for all current algorithms.
- **`--cpu` / `--num_cpu`**: launch with this many MPI-connected processes; `auto` uses all
available. Raises an error for algorithms that do not support parallelization.
- **`--dt` / `--datestamp`**: put timestamps in the save directory names.
- **User-supplied shorthand**: square brackets after a flag, e.g. `--hid[h]`, control the
directory-name abbreviation. Without one, a shorthand is derived automatically
(`clip_ratio` becomes `cli`).
- **`eval()` passthrough**: flag values pass through `eval()` before use, so you can name
functions and objects directly, e.g. `--act torch.nn.ELU`.
- **Dict kwargs**: `--key:v1 value_1 --key:v2 value_2` instead of `--key dict(v1=..., v2=...)`.
## Code Examples
The quickstart, with every flag doing something distinct:
```bash
python -m spinup.run ppo --exp_name ppo_ant --env Ant-v2 --clip_ratio 0.1 0.2 \
--hid[h] [32,32] [64,32] --act torch.nn.Tanh --seed 0 10 20 --dt \
--data_dir path/to/data
```
Choosing the backend explicitly:
```bash
python -m spinup.run ppo_pytorch --env Walker2d-v2 --exp_name walker
python -m spinup.run ppo_tf1 --env Walker2d-v2 --exp_name walker
# bare `ppo` reads spinup/user_config.py for the default backend
```
From a script:
```python
from spinup import ppo_pytorch as ppo
import gym
env_fn = lambda: gym.make('LunarLander-v2')
ac_kwargs = dict(hidden_sizes=[64, 64])
logger_kwargs = dict(output_dir='path/to/output_dir', exp_name='experiment_name')
ppo(env_fn=env_fn, ac_kwargs=ac_kwargs, steps_per_epoch=5000, epochs=250,
logger_kwargs=logger_kwargs)
```
ExperimentGrid, from `spinup/examples/pytorch/bench_ppo_cartpole.py`:
```python
from spinup.utils.run_utils import ExperimentGrid
from spinup import ppo_pytorch
import torch
eg = ExperimentGrid(name='ppo-pyt-bench')
eg.add('env_name', 'CartPole-v0', '', True)
eg.add('seed', [10*i for i in range(args.num_runs)])
eg.add('epochs', 10)
eg.add('steps_per_epoch', 4000)
eg.add('ac_kwargs:hidden_sizes', [(32,), (64,64)], 'hid')
eg.add('ac_kwargs:activation', [torch.nn.Tanh, torch.nn.ReLU], '')
eg.run(ppo_pytorch, num_cpu=args.cpu)
```
## Worked Example
`python -m spinup.run ddpg_tf1 --env Hopper-v2 --hid[h] [300] [128,128] --act tf.nn.tanh tf.nn.relu`
Two `hid` values times two `act` values is four experiments, run in series, producing four
suffixes:
```
_h128-128_ac-actrelu
_h128-128_ac-acttanh
_h300_ac-actrelu
_h300_ac-acttanh
```
`h` came from the user-supplied `[h]`; `ac-act` was derived automatically from the true flag
name `ac_kwargs:activation`. Add `--seed 0 10 20` and you get twelve runs in four folders —
three seeds grouped per configuration, which is the grouping the plotter averages over.
## Anti-patterns
- **Running the per-algorithm files directly** (`spinup/algos/BACKEND/ALGO/ALGO.py`). The
command-line support there is vestigial, takes a different argument set, and is explicitly
not the recommended way to run experiments.
- **Using ZShell without escaping square brackets**: ZShell treats them as special characters
and Spinning Up uses them for both list values and shorthands.
- **One seed.** RL algorithms have high variance; the flag exists so you use it (ch10 asks
for at least 3, ideally 10 or more).
- **Expecting parallel experiment launches.** Sweeps run in series by design.
## Key Takeaways
1. Every kwarg is a flag; every list of flag values is a sweep.
2. Sweeps run in series; MPI parallelism is per-experiment via `--cpu`, and only for
algorithms that support it.
3. Save-directory suffixes encode only the varying hyperparameters, never the seed — that
grouping is what makes multi-seed plotting work.
4. `ExperimentGrid` is the scripted form and loses only the shortcut kwargs.
5. Values go through `eval()`, so activations and other objects can be named on the command line.
## Connects To
- **Ch 5**: what the directories those flags create actually contain.
- **Ch 6**: the plotter, which consumes this directory structure and its autocompletion.
- **Ch 10**: why multi-seed runs are a rigor requirement, not a convenience.
- **Ch 20**: `ExperimentGrid`, `call_experiment` and `setup_logger_kwargs` in Run Utils.

97
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch05-experiment-outputs.md Normal file
View file

@ -0,0 +1,97 @@
# Chapter 5: Experiment Outputs
## Core Idea
Every run saves four things — a config record, a TSV of training metrics, the trained
model, and a pickled copy of the environment — and only one of them (`config.json`) is
ever meant to be read by hand.
## Frameworks Introduced
- **Tools, not files**: `test_policy` loads from `pyt_save/` or `tf1_save/`, the plotter
interprets `progress.txt`. Those are the correct interfaces. `config.json` is the only
file you should ever open yourself, and only to remember what you ran.
- When to use: any time you are tempted to parse a save directory manually.
- **Watch, then measure**: `test_policy` renders the agent so you can see behavior that no
scalar metric would have shown you; `-nr` drops rendering when you only want the numbers.
## Key Concepts
- **`progress.txt`**: tab-separated records of every metric the logger recorded, e.g. `Epoch`,
`AverageEpRet`.
- **`config.json`**: as-complete-as-possible dict of the args and kwargs used to launch the run.
Record-keeping only — launching an experiment from a config file is not supported.
- **`vars.pkl`**: algorithm state; currently used only to save a copy of the environment.
- **`pyt_save/model.pt`**: a pickled PyTorch `nn.Module`; loading restores an ActorCritic
object with an `act` method.
- **`tf1_save/`**: `variables/`, `model_info.pkl` (key-to-tensor-name map) and `saved_model.pb`.
Renamed from `simple_save/` on 2020-01-30.
- **`DEFAULT_DATA_DIR`**: set in `spinup/user_config.py`; defaults to `spinningup/data`.
## Reference Tables
| File | Contents |
|------|----------|
| `pyt_save/` | PyTorch only. Everything needed to restore the agent and value functions |
| `tf1_save/` | TF1 only. SavedModel plus the key-to-tensor map |
| `config.json` | The launch args and kwargs. Non-serializable values become strings |
| `progress.txt` | TSV of logged metrics across training |
| `vars.pkl` | Pickled environment copy; may be empty if the env cannot be pickled |
`test_policy` flags:
| Flag | Default | Does |
|------|---------|------|
| `-l L`, `--len=L` | 0 | Max episode length; 0 means no maximum |
| `-n N`, `--episodes=N` | 100 | Number of test episodes |
| `-nr`, `--norender` | off | Print returns and lengths only; much faster |
| `-i I`, `--itr=I` | -1 | Which saved snapshot; -1 is latest (see below) |
| `-d`, `--deterministic` | off | **SAC only.** Use the deterministic mean policy |
## Code Examples
```bash
python -m spinup.run test_policy path/to/output_directory
```
When the environment failed to pickle and `test_policy` raises `AssertionError: Environment
not found!`, rebuild it by hand:
```python
from spinup.utils.test_policy import load_policy_and_env, run_policy
import your_env
_, get_action = load_policy_and_env('/path/to/output_directory')
env = your_env.make()
run_policy(env, get_action)
```
To keep snapshots from many points in training (off by default — algorithms overwrite the
most recent one), change the line present in every algorithm:
```python
logger.save_state({'env': env}, None) # default: one snapshot, overwritten
logger.save_state({'env': env}, epoch) # keep per-epoch snapshots
```
and then set `save_freq` to something reasonable — at the default of 1 you flood the output
directory with one folder per epoch.
## Anti-patterns
- **Forgetting `-d` when evaluating SAC.** SAC trains a stochastic policy but the correct
evaluation policy is the deterministic mean. Without the flag you are measuring the wrong
thing, and the flag is used for no other algorithm.
- **Expecting to resume training.** Spinning Up implementations have no way to resume a
partially-trained agent.
- **Trying to relaunch from `config.json`.** It is a record, not an input.
- **Assuming `vars.pkl` has your environment.** Gym Box2D environments in older Gym versions
are known to fail to pickle, leaving it empty.
## Key Takeaways
1. Four artifacts per run; use the tools for three of them and read only `config.json` by hand.
2. SAC evaluation requires `--deterministic`; every other algorithm ignores it.
3. Multi-snapshot saving is a two-line change plus a `save_freq` you must set deliberately.
4. Environment pickling can silently fail — the recovery path is rebuilding the env in Python.
5. No resume support; a killed run is a lost run.
## Connects To
- **Ch 4**: the flags that create these directories.
- **Ch 6**: `progress.txt` is what the plotter reads.
- **Ch 19**: why SAC's evaluation policy differs from its training policy.
- **Ch 20**: the EpochLogger that writes all of this.

79
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch06-plotting-results.md Normal file
View file

@ -0,0 +1,79 @@
# Chapter 6: Plotting Results
## Core Idea
The plotter reads a directory prefix, autocompletes every matching run beneath it, averages
across seeds by default, and resolves the pseudo-metric `Performance` to the *correct*
performance measure for each algorithm family — so a fair on-policy versus off-policy
comparison is one command.
## Frameworks Introduced
- **`Performance` as a family-aware alias**: `Performance` is not a real output of any
algorithm. The plotter resolves it per logdir to `AverageEpRet` for on-policy algorithms
and `AverageTestEpRet` for off-policy ones.
- When to use: always, unless you specifically want one raw column. It is the default `-y`.
- Why it matters: on-policy performance is the average return of the batch just collected;
off-policy performance is measured by separate deterministic test rollouts. Plotting the
same raw column for both would compare two different quantities.
- **Prefix autocompletion**: logdirs are searched recursively and prefixes expand. Give the
plotter `data/bench_algo` and it finds `bench_algo1` and `bench_algo2` with all their seeds.
- How: name experiments with a shared prefix at launch and comparison becomes free.
- **Average by default, `--count` to disaggregate**: by default, y-values are averaged across
all results sharing an `exp_name` — typically identical experiments differing only in seed.
`--count` shows each curve separately.
## Key Concepts
- **`--xaxis` / `-x`**: which column is the x-axis. Default `TotalEnvInteracts`, i.e. sample
efficiency is the default framing, not wall-clock or epochs.
- **`--value` / `-y`**: which columns to graph; multiple values produce multiple graphs.
- **`--smooth S`**: average over a fixed window of width S. The book's own benchmark plots
use a window of 11 epochs (ch13).
- **`--legend` / `-l`**: overrides the automatic legend, which uses `exp_name` from
`config.json`. One string per *matched* directory, which may exceed the number of logdir
arguments you passed.
- **`--select` / `--exclude`**: keep or drop curves whose logdir contains all of / any of
these substrings. The way to narrow an over-eager autocomplete.
## Code Examples
```bash
python -m spinup.run plot [path/to/output_directory ...] [--legend [LEGEND ...]] \
[--xaxis XAXIS] [--value [VALUE ...]] [--count] [--smooth S] \
[--select [SEL ...]] [--exclude [EXC ...]]
```
Comparing two algorithms across all their seeds, relying on autocompletion:
```
data/
bench_algo1/
bench_algo1-seed0/
bench_algo1-seed10/
bench_algo2/
bench_algo2-seed0/
bench_algo2-seed10/
```
```bash
python spinup/utils/plot.py data/bench_algo
```
## Anti-patterns
- **Passing one legend string per logdir argument.** Autocompletion may match several
directories per argument; you need one legend entry per match, unless you narrowed the
set with `--select` / `--exclude`.
- **Plotting `AverageEpRet` for an off-policy algorithm** and calling it performance — that
is the behavior policy with exploration noise, not the evaluated policy. Use `Performance`.
- **Heavy smoothing to make a result look clean.** Smoothing is a display parameter; it does
not change what the seeds did, and ch10 asks you to show seed variance rather than hide it.
## Key Takeaways
1. `Performance` is the correct default because it is algorithm-family aware.
2. Prefix autocompletion plus shared `exp_name` prefixes is the intended comparison workflow.
3. Seed averaging is on by default; `--count` reveals the individual curves that average hides.
4. The default x-axis is environment interactions — sample efficiency is the default question.
## Connects To
- **Ch 5**: `progress.txt`, the plotter's input.
- **Ch 4**: the suffix rules that group seeds into one folder for averaging.
- **Ch 13**: the benchmark plots, produced with 10 seeds and an 11-epoch smoothing window.

139
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch07-key-concepts-in-rl.md Normal file
View file

@ -0,0 +1,139 @@
# Chapter 7: Part 1 — Key Concepts in RL
## Core Idea
RL is the study of agents learning by trial and error; formally, the agent maximizes
expected return J(pi) over trajectories, and almost every algorithm gets there through one
of four value functions, all of which obey Bellman self-consistency equations.
## Frameworks Introduced
- **The agent-environment interaction loop**: at every step the agent sees a (possibly
partial) observation, chooses an action, and receives a reward. The environment changes
because of the action, and may also change on its own.
- How: name the five MDP pieces before writing any code — states S, actions A, reward
function R, transition function P, start-state distribution rho_0.
- **The RL optimization problem**: `pi* = argmax_pi J(pi)`, where
`J(pi) = E_{tau ~ pi}[R(tau)]` and the trajectory distribution is
`P(tau|pi) = rho_0(s_0) * prod_t P(s_{t+1}|s_t,a_t) * pi(a_t|s_t)`.
- When to use: as the definition every algorithm is trying to approximate. If an algorithm
optimizes something else (a Bellman residual, a surrogate objective), that substitution
is the source of its failure modes.
- **The four value functions**:
- `V^pi(s)` — on-policy value: expected return starting in s and acting by pi forever.
- `Q^pi(s,a)` — on-policy action-value: same, but take an arbitrary a first.
- `V*(s)` — optimal value: expected return acting optimally from s.
- `Q*(s,a)` — optimal action-value: take arbitrary a, then act optimally.
- Two connections that come up constantly: `V^pi(s) = E_{a~pi}[Q^pi(s,a)]` and
`V*(s) = max_a Q*(s,a)`.
- **The optimal-action shortcut**: if you have `Q*`, you get the optimal policy for free:
`a*(s) = argmax_a Q*(s,a)`. There may be several maximizers, all optimal, but there is
always an optimal policy that picks deterministically.
- When to use: this is the entire justification for Q-learning as a family — learn `Q*`
and the policy is a lookup. Ch17 explains what breaks when the argmax is over a
continuous space.
- **Bellman equations**: "the value of your starting point is the reward you expect to get
from being there, plus the value of wherever you land next."
- On-policy: `V^pi(s) = E_{a~pi, s'~P}[r(s,a) + gamma V^pi(s')]`
- Optimal: `V*(s) = max_a E_{s'~P}[r(s,a) + gamma V*(s')]`
- The one crucial difference is the presence of the `max` over actions in the optimal form,
reflecting that an agent free to choose must pick the highest-value action.
- **Bellman backup** = the right-hand side, the reward-plus-next-value.
- **The advantage function**: `A^pi(s,a) = Q^pi(s,a) - V^pi(s)`. How much better taking a
specific action is than randomly selecting one according to pi, assuming you follow pi
afterwards.
- When to use: whenever relative quality is what matters and absolute value is noise.
Crucially important to policy gradient methods (ch9).
## Key Concepts
- **State vs observation**: a state s is a complete description of the world; an observation
o may omit information. Fully observed vs partially observed environments. Notation
routinely writes s where o is technically correct.
- **Action space**: the set of valid actions. **Discrete** (Atari, Go) versus **continuous**
(robot control, real-valued vectors). The distinction has profound consequences: some
algorithm families apply directly only to one case.
- **Policy**: deterministic `a_t = mu(s_t)` or stochastic `a_t ~ pi(.|s_t)`. In deep RL these
are **parameterized** — parameters theta or phi written as a subscript. "Policy" is often
used interchangeably with "agent."
- **Trajectory** (also **episode**, **rollout**): `tau = (s_0, a_0, s_1, a_1, ...)`.
- **Finite-horizon undiscounted return**: `R(tau) = sum_{t=0}^{T} r_t`.
- **Infinite-horizon discounted return**: `R(tau) = sum_{t=0}^{inf} gamma^t r_t`, with
`gamma` in (0,1).
- **MDP**: the 5-tuple `<S, A, R, P, rho_0>`. Markov property: transitions depend only on the
most recent state and action, not on prior history.
## Code Examples
A deterministic continuous-action policy is just an MLP:
```python
pi_net = nn.Sequential(
nn.Linear(obs_dim, 64), nn.Tanh(),
nn.Linear(64, 64), nn.Tanh(),
nn.Linear(64, act_dim)
)
obs_tensor = torch.as_tensor(obs, dtype=torch.float32)
actions = pi_net(obs_tensor)
```
**Categorical policies** (discrete actions) are built exactly like a classifier: observation
in, layers, a final linear layer giving logits per action, softmax to probabilities. Sampling
uses the framework's built-in categorical sampler. Log-likelihood is a vector index:
`log pi_theta(a|s) = log [P_theta(s)]_a`.
**Diagonal Gaussian policies** (continuous actions) always have a network mapping observations
to mean actions `mu_theta(s)`. The covariance is diagonal, so it is a vector, represented one
of two ways:
1. A single **state-independent** vector of log standard deviations — standalone parameters.
*Spinning Up's VPG, TRPO and PPO do it this way.*
2. A network `log sigma_theta(s)` mapping states to log standard deviations, optionally
sharing layers with the mean network. *SAC does it this way, and SAC with state-independent
log stds did not work (ch19).*
Log standard deviations, not standard deviations, because logs are free to range over
(-inf, inf) while stds must be nonnegative, and unconstrained parameters are easier to train.
Nothing is lost: exponentiate to recover.
Sampling: `a = mu_theta(s) + sigma_theta(s) * z` with `z ~ N(0, I)` (elementwise product).
Log-likelihood for a k-dimensional action:
`log pi_theta(a|s) = -0.5 * ( sum_i [ (a_i - mu_i)^2 / sigma_i^2 + 2 log sigma_i ] + k log 2pi )`
## Mental Models
- **A discount factor is two arguments in one**: intuitively, cash now beats cash later;
mathematically, an infinite sum of rewards may not converge and is hard to work with.
- **The formalism is starker than the practice.** Deep RL routinely sets up algorithms to
optimize the *undiscounted* return while using discount factors when *estimating value
functions*. Expect the line to be blurred in real code.
- Treat **Q\* as a policy in disguise** and **V\* as a scoring function**: only the action-value
form directly yields an action.
- Ask of every value function: **is it time-dependent?** Unless stated otherwise, value
functions mean infinite-horizon discounted return. Finite-horizon undiscounted value
functions would need time as an argument — because what a state is worth depends on how
much time is left.
## Anti-patterns
- **Reading `s` in an equation as a true state.** In partially observed settings the action
is conditioned on the observation; the notation is convention, not a claim.
- **Assuming an algorithm ports across action-space types.** Moving a method between discrete
and continuous spaces can require substantial rework, not a flag.
- **Learning `V*` and expecting to act.** Without `Q*` or a model you cannot extract the
action from the value.
## Key Takeaways
1. The goal is always `argmax_pi E_{tau~pi}[R(tau)]`; everything else is machinery for
approximating it.
2. Four value functions, two of them optimal; `V^pi = E_a[Q^pi]` and `V* = max_a Q*` connect them.
3. Bellman equations hold for all four; the `max` is what separates optimal from on-policy.
4. `a*(s) = argmax_a Q*(s,a)` is why Q-learning is a viable family at all.
5. Advantage `A = Q - V` is the relative-quality signal that policy gradients are built on.
6. Log standard deviations are parameterized, not standard deviations, and state-independence
is an implementation choice that differs across the algorithms in this book.
## Connects To
- **Ch 8**: the taxonomy built on what each algorithm chooses to learn.
- **Ch 9**: where the advantage function becomes the policy gradient weight.
- **Ch 17**: the continuous-action argmax problem, straight out of `a*(s) = argmax_a Q*(s,a)`.
- **Ch 12**: Exercise 1.1 asks you to implement the diagonal Gaussian log-likelihood above.

123
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch08-kinds-of-rl-algorithms.md Normal file
View file

@ -0,0 +1,123 @@
# Chapter 8: Part 2 — Kinds of RL Algorithms
## Core Idea
Two branching questions organize modern RL: does the agent have or learn a **model** of the
environment, and **what does it learn** — a policy, a Q-function, a value function, or a
model. Every trade-off in algorithm design descends from those two choices.
## Frameworks Introduced
- **Branch 1: model-free vs model-based.** A model is a function predicting state transitions
and rewards.
- **Upside of a model: it allows the agent to plan** — think ahead, see what would happen
across options, explicitly decide, then distil the planning results into a learned policy.
AlphaZero is the famous case. When it works, it substantially improves sample efficiency.
- **Downside: a ground-truth model is usually not available**, so it must be learned from
experience, and **bias in a learned model gets exploited by the agent** — producing an
agent that performs well against its own model and sub-optimally, or terribly, in the
real environment. Model-learning is fundamentally hard; large amounts of time and compute
can fail to pay off.
- When to use model-based: you have a reliable model, or sample efficiency dominates and
you can afford the model-bias risk.
- **Branch 2: what to learn in model-free RL — two families.**
- **Policy Optimization.** Represent the policy explicitly as `pi_theta(a|s)` and optimize
theta either directly by gradient ascent on `J(pi_theta)` or indirectly by maximizing
local approximations of it. Almost always **on-policy**. Usually also learns an
approximator `V_phi(s)` used in figuring out the policy update. Examples: A2C/A3C
(direct), PPO (indirect, via a surrogate objective giving a conservative estimate of how
much `J` will change).
- **Q-Learning.** Learn an approximator `Q_theta(s,a)` to `Q*(s,a)`, usually with an
objective based on the Bellman equation. Almost always **off-policy**. The policy comes
from the connection `a(s) = argmax_a Q_theta(s,a)`. Examples: DQN, C51 (which learns a
distribution over return whose expectation is `Q*`).
- **The policy-optimization vs Q-learning trade-off** (the single most quoted judgment in
this book):
- Policy optimization is **principled — you directly optimize for the thing you want** —
which tends to make it stable and reliable.
- Q-learning only *indirectly* optimizes agent performance, by training `Q_theta` to satisfy
a self-consistency equation. **There are many failure modes for this kind of learning, so
it tends to be less stable.**
- But when Q-learning works it is **substantially more sample efficient**, because it can
reuse data far more effectively.
- How to use: pick the failure you can detect and afford. Instability you can see in a
learning curve; sample inefficiency you can see in a compute bill.
- **The interpolation principle**: policy optimization and Q-learning are not incompatible,
and under some circumstances turn out to be *equivalent* (Schulman et al 2017). A range of
algorithms lives between them and trades off deliberately: **DDPG** (learns a deterministic
policy and a Q-function that improve each other) and **SAC** (a variant using stochastic
policies, entropy regularization and other tricks to stabilize learning; scores higher than
DDPG on standard benchmarks).
- **Four ways to use a model** (there is no small set of clean clusters here; the model may
be given or learned in each):
1. **Pure planning** — never represent the policy at all. Model-predictive control (MPC):
each time the agent observes, compute a plan optimal with respect to the model over a
fixed window, execute only the first action, discard the rest, re-plan next step. Future
rewards past the horizon can enter through a learned value function. Example: MBMF.
2. **Expert iteration** — keep an explicit policy `pi_theta`, use a planning algorithm
(e.g. Monte Carlo Tree Search) inside the model with candidate actions sampled from the
current policy. The planner's output is an "expert" relative to the policy; update the
policy toward it. Examples: ExIt, AlphaZero.
3. **Data augmentation for model-free methods** — train a policy or Q-function with a
model-free algorithm, but augment real experience with fictitious experience (MBVE), or
train on purely fictitious experience ("training in the dream", World Models).
4. **Embedding planning loops into policies** — make the planning procedure a subroutine of
the policy so complete plans become side information, and train the policy output with
any standard model-free algorithm. **The key advantage: model bias becomes less of a
problem, because where the model is bad for planning the policy can learn to ignore it.**
Example: I2A.
## Key Concepts
- **Model of the environment**: a function predicting state transitions and rewards.
- **Surrogate objective**: a local approximation of `J(pi_theta)` that is safe to maximize;
the mechanism behind TRPO and PPO.
- **The deadly triad**: function approximation + bootstrapping + off-policy data, which
together cause instability in value-learning algorithms (Sutton and Barto ch. 11.3). This
is the concrete content of "Q-learning has many failure modes."
- **Modularity caveat**: the taxonomy is a tree and the real space is not. Advanced areas —
exploration, transfer learning, meta learning — are omitted from it entirely.
## Reference Tables
| Choice | Buys you | Costs you |
|--------|----------|-----------|
| Model-based | Planning; large sample-efficiency gains when the model is good | Model must usually be learned; model bias gets exploited |
| Model-free | Easier to implement and tune; more developed and tested | Forfeits the sample-efficiency gains a model could give |
| Policy optimization | Stability, reliability, directly optimizes performance | Sample inefficiency (on-policy data only) |
| Q-learning | Substantial sample efficiency through data reuse | Instability; no guarantee good Bellman fit means good policy |
## Mental Models
- **Place any new algorithm with two questions**: model or no model, and what does it learn.
That is what the taxonomy is for; it does not need to be exhaustive to do that job.
- **Model bias is an adversarial problem, not a noise problem.** The agent is actively
optimizing against your model's errors. That is why "the model is only 95% accurate" does
not translate into "the policy is 95% as good."
- **"Directly optimize the thing you want" is a design principle you can apply outside RL**:
the further your training objective sits from your evaluation metric, the more failure modes
you inherit.
## Anti-patterns
- **Reading the taxonomy as exhaustive or as a strict tree.** The book opens with that
disclaimer: algorithm modularity is not well represented by a tree.
- **Assuming model-based is strictly better because it plans.** As of the guide's writing
(September 2018), model-free methods were more popular and more extensively developed and
tested, precisely because model-learning is hard.
- **Treating "it fits the Bellman equation well" as evidence the policy is good.** No such
guarantee exists — this is the defining weakness of the whole Q-learning family.
## Key Takeaways
1. Two branching questions — model or not, and what to learn — generate the whole landscape.
2. Policy optimization: principled, stable, sample-hungry. Q-learning: sample-efficient,
indirect, unstable.
3. The two families are not exclusive; DDPG and SAC live between them on purpose.
4. Model bias is exploited by the agent, which is the central risk of model-based methods.
5. Embedding a planner inside a policy is the model-based approach that most directly
defuses model bias, because the policy can learn when to ignore the model.
## Connects To
- **Ch 3**: the six implemented algorithms, which are all model-free.
- **Ch 9**: the mathematics of the policy-optimization family.
- **Ch 17 and Ch 19**: DDPG and SAC as the concrete interpolations named here.
- **Ch 11**: the key-papers list, whose top-level sections mirror this taxonomy.

161
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch09-intro-to-policy-optimization.md Normal file
View file

@ -0,0 +1,161 @@
# Chapter 9: Part 3 — Intro to Policy Optimization
## Core Idea
Three results build the policy gradient you actually implement: the simplest analytical
expression for `grad J`, a rule that lets you **drop useless terms** (reward-to-go), and a
rule that lets you **add useful terms** (baselines) — ending at the advantage-weighted form
`grad J = E[ sum_t grad log pi_theta(a_t|s_t) * A^pi(s_t,a_t) ]`.
## Frameworks Introduced
- **The general policy gradient form**: every variant is
`grad_theta J(pi_theta) = E_{tau~pi_theta}[ sum_{t=0}^{T} grad_theta log pi_theta(a_t|s_t) * Phi_t ]`
and the whole subject is *which* `Phi_t` you choose. Five valid choices, all with the same
expectation and different variance:
1. `Phi_t = R(tau)` — the full trajectory return (the simplest form)
2. `Phi_t = sum_{t'=t}^{T} R(s_t', a_t', s_{t'+1})` — the **reward-to-go**
3. `Phi_t = reward-to-go - b(s_t)` — reward-to-go with a **baseline**
4. `Phi_t = Q^{pi_theta}(s_t, a_t)`
5. `Phi_t = A^{pi_theta}(s_t, a_t)` — the **advantage**, the one Spinning Up's VPG uses
- When to use: choice 5 in practice; the list is the map for reading any policy gradient paper.
- **The derivation, in five facts.** Worth memorizing because every extension reuses them:
1. `P(tau|theta) = rho_0(s_0) * prod_t P(s_{t+1}|s_t,a_t) * pi_theta(a_t|s_t)`
2. **Log-derivative trick**: `grad_theta P(tau|theta) = P(tau|theta) * grad_theta log P(tau|theta)`
3. `log P(tau|theta) = log rho_0(s_0) + sum_t [ log P(s_{t+1}|s_t,a_t) + log pi_theta(a_t|s_t) ]`
4. **Gradients of environment functions are zero** — the environment has no dependence on theta,
so `rho_0`, `P` and `R` all vanish under `grad_theta`.
5. Therefore `grad_theta log P(tau|theta) = sum_t grad_theta log pi_theta(a_t|s_t)`.
- Chained: expand the expectation, bring the gradient inside the integral, apply the
log-derivative trick, return to expectation form, substitute fact 5.
- **The EGLP lemma (Expected Grad-Log-Prob)**: for any parameterized distribution `P_theta`,
`E_{x~P_theta}[ grad_theta log P_theta(x) ] = 0`.
- Proof in three lines: all distributions are normalized (`integral P_theta(x) = 1`); take
the gradient of both sides (`= grad 1 = 0`); apply the log-derivative trick.
- When to use: it is the engine behind both the drop rule and the add rule. The author notes
it has no standard name in the literature but comes up often enough to deserve one.
- **"Don't let the past distract you" (the drop rule)**: agents should only reinforce actions
on the basis of their **consequences**. Rewards obtained *before* an action have no bearing
on how good that action was. Formally, all terms with `t' < t` are zero in expectation.
- Why it is better, precisely: those dropped terms had **zero mean but nonzero variance** —
they added pure noise to the sample estimate. Removing them reduces the number of sample
trajectories needed.
- **Baselines (the add rule)**: an immediate consequence of EGLP is that for any function `b`
depending only on state, `E_{a_t~pi}[ grad log pi(a_t|s_t) * b(s_t) ] = 0`. So you may add
or subtract any such term without changing the gradient in expectation.
- **The most common baseline is the on-policy value function `V^pi(s_t)`**, which empirically
reduces variance and gives faster, more stable learning.
- The conceptual appeal: it encodes the intuition that **if an agent gets what it expected,
it should "feel" neutral about it.**
- In practice `V^pi` cannot be computed exactly, so it is approximated by a network `V_phi`
updated concurrently with the policy (so it always approximates the *most recent* policy).
The simplest learning rule, used by VPG, TRPO, PPO and A2C, is mean-squared error:
`phi_k = argmin_phi E_{s_t, Rhat_t ~ pi_k}[ (V_phi(s_t) - Rhat_t)^2 ]`, via one or more
gradient steps starting from `phi_{k-1}`.
## Key Concepts
- **Policy gradient**: `grad_theta J(pi_theta)`. Algorithms that optimize this way are
policy gradient algorithms — VPG and TRPO are; **PPO is often called one though this is
slightly inaccurate.**
- **Reward-to-go**: `Rhat_t = sum_{t'=t}^{T} R(s_t', a_t', s_{t'+1})`.
- **Baseline**: any state-only function subtracted from the weight.
- **Sample estimate**: `ghat = (1/|D|) * sum_{tau in D} sum_t grad log pi_theta(a_t|s_t) * Phi_t`.
- **GAE (Generalized Advantage Estimation)**: the widely-used method for approximating the
advantage function; Spinning Up's VPG, TRPO and PPO all use it. The book strongly advises
studying the paper.
- **Epoch** (in this context): one experience-collection phase plus one policy gradient update.
## Code Examples
The whole simple algorithm is 128 lines (`spinup/examples/pytorch/pg_math/1_simple_pg.py`).
The three pieces that matter:
```python
# 1. Policy network
logits_net = mlp(sizes=[obs_dim] + hidden_sizes + [n_acts])
def get_policy(obs):
return Categorical(logits=logits_net(obs))
def get_action(obs):
return get_policy(obs).sample().item()
# 2. The "loss" whose gradient is the policy gradient
def compute_loss(obs, act, weights):
logp = get_policy(obs).log_prob(act)
return -(logp * weights).mean()
# 3. One gradient step
optimizer.zero_grad()
batch_loss = compute_loss(obs=..., act=..., weights=...)
batch_loss.backward()
optimizer.step()
```
Upgrading to reward-to-go changes one function and two lines:
```python
def reward_to_go(rews):
n = len(rews)
rtgs = np.zeros_like(rews)
for i in reversed(range(n)):
rtgs[i] = rews[i] + (rtgs[i+1] if i+1 < n else 0)
return rtgs
# was: batch_weights += [ep_ret] * ep_len
batch_weights += list(reward_to_go(ep_rews))
```
For a diagonal Gaussian policy, `log_prob(act)` returns per-component log probabilities of
shape `(batch, act_dim)` when RL needs shape `(batch,)`. Sum them:
```python
logp = get_policy(obs).log_prob(act).sum(axis=-1)
```
## Worked Example
**The policy gradient "loss" is not a loss function.** This is the single most consequential
warning in the chapter, and it differs from supervised learning in two ways:
1. **The data distribution depends on the parameters.** A supervised loss is defined on a
fixed data distribution independent of the parameters being optimized. Here the data must
be sampled from the most recent policy.
2. **It does not measure performance.** We care about `J(pi_theta)`, and this "loss" does not
approximate it — not even in expectation. It is useful only because, *evaluated at the
current parameters with data generated by those parameters*, it has the negative gradient
of performance.
After the first gradient step there is no connection to performance at all. You can send
this loss to negative infinity while policy performance craters — and it usually will.
Researchers sometimes call this the policy "overfitting" to a batch; the phrase is
descriptive but should not be taken literally, since it does not refer to generalization error.
**The practical rule: in policy gradients, only average return means anything. The loss
function means nothing.** The ML habit of reading "loss went down, all is well" is wrong here.
## Anti-patterns
- **Using the loss curve as a training health signal.** See above. Watch `AverageEpRet`.
- **Weighting every action by the full-trajectory return** when reward-to-go is a two-line
change that strictly reduces variance.
- **Forgetting to sum log-probabilities across action dimensions** for Gaussian policies —
a shape bug that will not raise, in the same family as the ch12 DDPG bug.
- **Treating a baseline as a bias**: baselines are provably zero-mean under EGLP. They change
variance, never the expected gradient.
## Key Takeaways
1. One template, five valid weights `Phi_t`; the advantage form is the destination.
2. The whole derivation rests on the log-derivative trick plus the fact that environment
gradients vanish.
3. EGLP is a single lemma that licenses both dropping past rewards and adding baselines.
4. Reward-to-go removes terms with zero mean and nonzero variance — pure noise reduction.
5. The value baseline is learned by mean-squared regression onto reward-to-go, concurrently
with the policy.
6. The policy gradient loss is not a performance measure and must never be read as one.
## Connects To
- **Ch 7**: the advantage function and value functions this chapter puts to work.
- **Ch 14**: VPG, the direct implementation of the final advantage-weighted form.
- **Ch 15 and Ch 16**: TRPO and PPO, which replace the plain gradient step with a
trust-region-constrained one.
- **Ch 12**: Exercise 2.1 measures exactly what happens when `V_phi` is not fit properly.

133
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch10-spinning-up-as-a-researcher.md Normal file
View file

@ -0,0 +1,133 @@
# Chapter 10: Spinning Up as a Deep RL Researcher
## Core Idea
Joshua Achiam's curriculum for becoming a deep RL researcher, in four stages: build the
right background, learn by writing your own implementations, develop a research project
through one of three idea frames, and hold yourself to experimental standards strict enough
that a positive result means something.
## Frameworks Introduced
- **The Right Background** (four items, all prerequisites, none optional):
- **Math**: from probability and statistics — random variables, Bayes' theorem, chain rule
of probability, expected values, standard deviations, importance sampling. From
multivariate calculus — gradients, and optionally Taylor series expansions.
- **General deep learning**: standard architectures (MLP, vanilla RNN, LSTM, GRU, conv
layers, resnets, attention), regularizers (weight decay, dropout), normalization (batch,
layer, weight norm), optimizers (SGD, momentum SGD, Adam), and the reparameterization trick.
- **One deep learning library**, well enough to confidently implement a simple supervised
learning program.
- **RL concepts and terminology**: states, actions, trajectories, policies, rewards, value
functions, action-value functions. Optionally, monotonic improvement theory (the basis
for advanced policy gradient algorithms) and classical RL algorithms.
- **Learn by Doing** — the core of the essay, in eight rules:
1. **Write your own implementations**, aiming for the **shortest correct implementation** of
each. By far the best way to build both understanding and performance intuitions.
2. **Simplicity is critical.** Implement the simplest algorithms first and add complexity
gradually. Starting with too many moving parts means weeks lost to debugging — a common
failure mode for newcomers. If you are stuck in it, drop to a simpler algorithm and
return later.
3. **Which algorithms, in roughly this order**: VPG (REINFORCE), DQN, A2C (the synchronous
A3C), PPO (clipped objective), DDPG. The simplest versions are a few hundred lines
(ballpark 250-300); a no-frills VPG is about 80. Write single-threaded before parallel,
but do parallelize at least one.
4. **Focus on understanding, because broken RL code almost always fails silently** — the
code runs fine and the agent simply never learns. Usually something is computed with the
wrong equation, on the wrong distribution, or piped to the wrong place. Sometimes the
only way to find it is to read the code critically, knowing exactly what it should do.
5. **What to look for in papers**: scour ablation analyses (they tell you which parameters
and subroutines actually matter, which is what you need to diagnose bugs) and
supplementary material (network architectures, optimization hyperparameters — align your
implementation to these to improve your odds of getting it working).
6. **But do not overfit to paper details.** Papers often prescribe more tricks than are
strictly necessary. The original DDPG paper suggests a complex architecture, an
initialization scheme and batch normalization; none are strictly necessary and some of
the best DDPG results use simpler networks. The original A3C uses asynchronous updates
from actor-learners; synchronous updates work about as well.
7. **Do not overfit to existing implementations either.** RL libraries make abstraction
choices that are good for code reuse across algorithms but unnecessary if you are
writing one algorithm for one use case.
8. **Iterate fast in simple environments.** CartPole-v0, InvertedPendulum-v0, FrozenLake-v0,
HalfCheetah-v2 with a short horizon (100 or 250 steps, not the full 1000).
**Ideal debug-stage turnaround: under 5 minutes on your local machine.** These runs need
no special hardware and run fine on CPUs. Do not attempt Atari or Humanoid before the
simplest toy task works.
- **Three idea-generation frames** — the frame you pick shapes the project's scope and risk:
| Frame | What it is | Scope | Main risk |
|-------|-----------|-------|-----------|
| **1. Improving on an existing approach** | Incrementalism: performance gains in an established setting by tweaking an existing algorithm. Reimplementing prior work exposes where it is brittle. Most accessible to novices, still worthwhile at any level | Narrow, wraps up in a few months | Your tweaks may simply fail to improve it, and then the project is over with no signal on what to do next |
| **2. Focusing on unsolved benchmarks** | Succeed at a task nobody has solved. You may try a wide variety of methods, prior and invented | Broad, several months to a year-plus | The benchmark may be unsolvable without a substantial breakthrough; easy to spend a long time with no progress. Even failure usually yields insights that seed the next project |
| **3. Creating a new problem setting** | An entirely different conceptual problem nobody has studied; you will have to design the benchmark too | Open-ended | Enormous challenge — but great benchmarks move the whole field forward. These problems come up when they come up; hard to go looking for them |
- **Doing Rigorous Research in RL** — four standards, each closing a specific way a result
can look real and not be:
1. **Set up fair comparisons.** If you implement your baseline from scratch, spend **as much
time tuning the baseline as tuning your own algorithm.** Hold all else equal even when
the methods differ substantially — e.g. keep parameter counts approximately equal when
investigating architecture variants. **Under no circumstances handicap the baseline.**
RL baselines are strong, and consistent wins over them are hard.
2. **Remove stochasticity as a confounder.** Run everything for many random seeds — **at
least 3, and 10 or more to be thorough.** Deep RL is fairly brittle with respect to seed;
two different groups of seeds can produce learning curves so different they look like
they come from different distributions.
3. **Run high-integrity experiments.** Do not report the best or most interesting runs.
Launch **new, final experiments** for every method being compared and **precommit to
reporting whatever comes out.** This enforces a weak form of preregistration: the tuning
stage produces your hypotheses, the final runs produce your conclusions.
4. **Check each claim separately — run an ablation analysis.** Any proposed method has
several key design decisions, and the claim "these collectively help" is really a bundle
of separate claims. Systematically swap or remove each one to attribute credit correctly.
This lets you state each claim with a measure of confidence and strengthens the whole work.
## Key Concepts
- **Silent failure**: the defining property of broken RL code. It runs; the agent just never learns.
- **Measure everything**: instrument heavily. The author's own list — mean/std/min/max of
cumulative rewards, episode lengths and value function estimates, plus the objective losses
and any exploration parameters (mean policy entropy for stochastic policy optimization,
current epsilon for epsilon-greedy). **Also watch videos of your agent** periodically; it
gives insights nothing else does. You cannot tell it is broken if you cannot see it breaking.
- **Assume there is a bug.** Spend a lot of effort searching for bugs before tweaking
hyperparameters. Bad hyperparameters can significantly degrade performance, but if yours
are similar to those in papers and standard implementations, they are probably not the issue.
- **Test in more than one environment**: sometimes code works in one environment despite a
breaking bug, so re-verify once results look promising.
- **Avoid reinventing the wheel**: before investing, check thoroughly that the idea has not
been done. But do not let the risk push you into planting flags with not-quite-finished
research or over-claiming partial work. Complete, thorough investigations are what counts.
- **Scale when things work**: after the simplest environments pass, move to harder ones —
experiments now take hours to a couple of days, and specialized hardware (a beefy GPU, a
32-core machine) or cloud resources start to be worth it.
## Anti-patterns
- **Tuning hyperparameters to fix a bug.** It is usually a bug.
- **Debugging in a hard environment.** If turnaround exceeds a few minutes at the debug stage,
you are working in the wrong environment.
- **Starting with the complex algorithm.** The predicted outcome is weeks lost.
- **Reporting your best run.** That is the failure the precommitment rule exists to stop.
- **Under-tuning the baseline** — the most common way to manufacture a positive result without
intending to.
- **Fewer than 3 seeds.** With deep RL's seed sensitivity, a single-seed result carries
almost no information.
- **Bundling claims.** Without ablations you cannot say which of your design decisions did
the work — including, possibly, none of them.
## Key Takeaways
1. Implement from scratch, simplest first, shortest correct version, single-threaded before parallel.
2. Broken RL code fails silently — so instrumentation and critical reading, not error messages,
are your debugging tools.
3. Debug loops under five minutes in toy environments; scale only after correctness.
4. Read papers for ablations and supplementary material, but do not adopt every trick.
5. Pick an idea frame deliberately: incremental (narrow, fast, may dead-end), unsolved
benchmark (broad, slow, insight-rich even in failure), or new problem setting (rare, huge).
6. Rigor is four things: a fully-tuned baseline, many seeds, precommitted final runs, and
per-claim ablations.
7. These habits are worth keeping past the learning stage; they accelerate research.
## Connects To
- **Ch 1**: the code design philosophy that makes the reference implementations readable.
- **Ch 4**: the `--seed 0 10 20` flag that operationalizes the multi-seed rule.
- **Ch 11**: the key papers list, the recommended starting point for literature exploration.
- **Ch 12**: the exercises — Problem Set 2 is entirely about silent failure modes.
- **Ch 13**: the benchmarks, run at 10 seeds — the standard this chapter asks for.

70
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch11-key-papers-in-deep-rl.md Normal file
View file

@ -0,0 +1,70 @@
# Chapter 11: Key Papers in Deep RL
## Core Idea
A curated, explicitly non-comprehensive reading list of roughly 100 papers organized into
13 topic areas — designed as a starting point for someone looking to do research, and as
the map you use to pick a research topic in ch10's "explore the literature" step.
## Frameworks Introduced
- **Use the list as a topic map, not a queue.** The intended workflow (from ch10): scan the
sections to become aware of what topics exist, find a paper on one that inspires you, read
it thoroughly, then use its related-work section and citations to do a deep dive. You will
start to see where the unsolved problems are.
- **Each entry is annotated with what it contributes** — either **Algorithm: X** (this paper
introduced X) or **Contribution: ...** (this paper established a result, a critique or a
codebase). Scan the annotations, not the titles.
## Reference Tables
The 13 sections, with the subsections that reveal the field's internal structure:
| # | Section | Subsections / notable entries |
|---|---------|------------------------------|
| 1 | **Model-Free RL** | a. Deep Q-Learning (DQN, Deep Recurrent Q-Learning, Dueling DQN, Double DQN, Prioritized Experience Replay, Rainbow) · b. Policy Gradients (A3C, TRPO, GAE, PPO, ACKTR, ACER, SAC) · c. Deterministic Policy Gradients (DPG, DDPG, TD3) · d. Distributional RL (C51, QR-DQN, IQN, Dopamine) · e. Policy Gradients with Action-Dependent Baselines (Q-Prop, Stein Control Variates, and Tucker et al's critique) · f. Path-Consistency Learning (PCL, Trust-PCL) · g. Other Ways of Combining Policy-Learning and Q-Learning (PGQL, Reactor, IPG, the policy-gradient/soft-Q-learning equivalence) · h. Evolutionary Algorithms (ES) |
| 2 | **Exploration** | a. Intrinsic Motivation (VIME, count-based pseudocounts, hash-based counts, EX2, ICM, RND) · b. Unsupervised RL (VIC, DIAYN, VALOR) |
| 3 | **Transfer and Multitask RL** | Progressive Networks, UVFA, UNREAL, PathNet, MATL, HER |
| 4 | **Hierarchy** | STRAW, Feudal Networks, HIRO |
| 5 | **Memory** | MFEC, NEC, Neural Map, MERLIN, RMC |
| 6 | **Model-Based RL** | a. Model is Learned (I2A, MBMF, MVE) · b. Model is Given (AlphaZero and relatives) |
| 7 | **Meta-RL** | |
| 8 | **Scaling RL** | |
| 9 | **RL in the Real World** | |
| 10 | **Safety** | |
| 11 | **Imitation Learning and Inverse Reinforcement Learning** | |
| 12 | **Reproducibility, Analysis, and Critique** | |
| 13 | **Bonus: Classic Papers in RL Theory or Review** | |
## Mental Models
- **Section 1's shape mirrors ch8's taxonomy**: Q-learning, policy gradients, and the
deterministic and interpolating methods between them. If you understood ch8, section 1
is already organized in your head.
- **Section 12 is the one people skip and shouldn't.** "Reproducibility, Analysis, and
Critique" is where ch10's rigor standards come from, and where you learn that published
gains sometimes do not survive re-examination — the Tucker et al entry in section 1e is a
worked example: it critiques and re-evaluates claims from earlier papers (including Q-Prop
and Stein control variates) and finds important methodological errors in them.
- **A reading list is a topic-awareness tool.** The list explicitly does not claim
completeness; its job is to prevent you from picking a research problem while unaware that
a whole subfield already works on it.
## Anti-patterns
- **Reading it front to back.** It is far from comprehensive and is not a curriculum; the
intended use is targeted depth after topic selection.
- **Treating an entry as endorsement of the result.** Several entries exist precisely because
they critique other entries.
- **Skipping the related-work walk.** The list is the entry point; the citations are the
actual literature review.
## Key Takeaways
1. 13 topic sections; section 1 (Model-Free RL) alone has eight subsections and mirrors ch8.
2. Every entry is annotated with the algorithm it introduced or the contribution it made.
3. The list is a topic map for choosing a research direction, not a reading queue.
4. Sections 7-13 (meta-RL, scaling, real world, safety, imitation/IRL, critique, classics)
are where the topics ch10 suggests for project ideas actually live.
## Connects To
- **Ch 8**: the taxonomy that section 1's structure reproduces.
- **Ch 10**: "start by exploring the literature to become aware of topics in the field" —
this chapter is that step's tool.
- **Ch 14-19**: every implemented algorithm's own "Why These Papers?" section is a
three-paper version of this list, scoped to one algorithm.

105
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch12-exercises.md Normal file
View file

@ -0,0 +1,105 @@
# Chapter 12: Exercises
## Core Idea
Two problem sets with opposite purposes: Problem Set 1 makes you write the mathematical
core of three algorithms, and Problem Set 2 makes you **watch RL code fail silently** and
find out why — because that is the skill the essay says decides whether you can do this work.
## Frameworks Introduced
- **Problem Set 1: Basics of Implementation** — you are given everything except the math.
- **1.1 Gaussian Log-Likelihood.** Write a function taking means, log stds and samples,
returning the log likelihoods. Auto-checked against a known-good implementation on a
batch of random inputs. The formula is the one in ch7.
- **1.2 Policy for PPO.** Implement an MLP diagonal Gaussian policy. Auto-checked by
running 20 epochs on InvertedPendulum-v2 (3-5 minutes). **Bar for success: average score
over 500 in the last 5 epochs, or 1000 (the maximum) in the last 5.**
- **1.3 Computation Graph for TD3.** You are given the entire TD3 algorithm *except* the
loss functions and the intermediate calculations for them. Find "YOUR CODE HERE".
No automatic checking. Evaluated on HalfCheetah-v2, InvertedPendulum-v2 and one other
MuJoCo environment of your choosing, with smaller networks ([128,128]), max episode
length 150, 10 epochs, roughly 10 minutes. **Anecdotal targets within 10 epochs:
HalfCheetah over 300, InvertedPendulum maxing out at 150.** `--use_soln` runs Spinning
Up's TD3 instead of yours.
- **Problem Set 2: Algorithm Failure Modes** — the point is the failure, not the fix.
- **2.1 Value Function Fitting in TRPO.** Compare `train_v_iters=80` against
`train_v_iters=0` on Hopper-v2, three seeds each, 250 epochs, 4000 steps per epoch.
**Result: the difference is substantial. With a trained value function the agent makes
quick progress; with an untrained one it gets stuck early on.** Few factors affect policy
gradient performance more drastically than the quality of the value function used for
advantage estimation.
- **2.2 Silent Bug in DDPG.** Run DDPG with and without a planted bug, three seeds each,
six runs, ~10 minutes each; plot and compare. Then, **without looking at DDPG's `core.py`**,
work out what the bug is.
## Worked Example
**The 2.2 bug, and why it is the most instructive page in the book.**
The correct and bugged actor-critic differ in exactly one thing: whether the Q-function
output is squeezed.
```python
# Correct
def forward(self, obs, act):
q = self.q(torch.cat([obs, act], dim=-1))
return torch.squeeze(q, -1) # Critical to ensure q has right shape.
# Bugged
def forward(self, obs, act):
return self.q(torch.cat([obs, act], dim=-1)) # shape [batch, 1], not [batch]
```
The TF1 version is the same defect: the correct code squeezes to shape `[batch size]`, the
bugged code leaves shape `[batch size, 1]`.
Why that one missing squeeze destroys learning — look at the DDPG graph:
```python
backup = tf.stop_gradient(r_ph + gamma*(1-d_ph)*q_pi_targ)
pi_loss = -tf.reduce_mean(q_pi)
q_loss = tf.reduce_mean((q - backup)**2)
```
`r_ph` and `d_ph` have shape `[batch size]`. The backup line was written assuming it adds
tensors of the same shape. But it will also happily add tensors of *different* shapes as long
as they are broadcast-compatible — and `[batch size]` and `[batch size, 1]` are compatible,
with results that are not what you expect:
```
x has shape [5], y has shape [5,1]
x * y -> shape [5,5]
x + y -> shape [5,5]
```
Adding or multiplying a shape-`[5]` tensor by a shape-`[5,1]` tensor returns a shape-`[5,5]`
tensor. So when the Q-functions are not squeezed, `q_pi_targ` has shape `[batch size, 1]`,
the backup becomes a `[batch, batch]` matrix, and the whole Q-loss is meaningless. **Nothing
raises. Nothing warns. The run completes. The agent just learns worse.**
This is ch10's "broken RL code almost always fails silently" reduced to a single missing
`squeeze`. The **Bonus** question is worth sitting with: *are there any choices of
hyperparameters which would have hidden the effects of the bug?*
## Anti-patterns
- **Looking at `core.py` before attempting 2.2.** The exercise is diagnostic practice; the
answer is worth less than the search.
- **Broadcast-compatible shapes as an implicit contract.** The lesson generalizes far past
DDPG: any elementwise op between a `[N]` and an `[N,1]` tensor is a silent bug waiting.
Assert shapes.
- **Skipping Problem Set 2 because it produces no code.** 2.1 produces no code at all and is
still one of the most decision-relevant results in the book.
## Key Takeaways
1. Problem Set 1 is math-in-code: Gaussian log-likelihood, a PPO policy, TD3's losses.
2. 1.1 and 1.2 self-check; 1.3 does not, and is evaluated on visible learning progress.
3. 2.1: a badly-fit value function does not degrade a policy gradient agent gracefully — it
gets stuck early.
4. 2.2: one missing `squeeze` silently corrupts the Bellman backup through broadcasting.
5. The challenges past the problem sets are: reimplement algorithms from scratch (ch10), and
attempt OpenAI's standing Requests for Research.
## Connects To
- **Ch 7**: the diagonal Gaussian log-likelihood formula that Exercise 1.1 asks for.
- **Ch 9**: why the value function baseline quality drives policy gradient performance (2.1).
- **Ch 17 and Ch 18**: the DDPG graph and the TD3 losses the exercises operate on.
- **Ch 10**: "broken RL code almost always fails silently" — Problem Set 2 is the proof.

87
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch13-benchmarks.md Normal file
View file

@ -0,0 +1,87 @@
# Chapter 13: Benchmarks for Spinning Up Implementations
## Core Idea
All six algorithms were benchmarked for 3M timesteps across five MuJoCo environments at
10 seeds each — and the results carry an explicit, unusual disclosure about which
implementations are research-grade and which are not.
## Frameworks Introduced
- **The parity disclosure — the single most decision-relevant paragraph in the chapter:**
- **DDPG, TD3 and SAC are roughly at parity with the best reported results** for those
algorithms. **You can use them for research purposes.**
- **VPG, TRPO and PPO are overall a bit weaker than the best reported results**, because
the implementations omit standard tricks — specifically **observation normalization** and
**normalized value regression targets**. **For research comparisons, use the TRPO or PPO
implementations from OpenAI Baselines instead.**
- When to use: before citing any Spinning Up number, or before using one of these as a
baseline in a paper (which ch10 says you must tune as hard as your own method).
- **Report performance with its measurement definition attached.** The two families are not
measured the same way, and saying "return" without saying which is a category error:
- **On-policy**: the average trajectory return across the batch collected at each epoch —
i.e. the exploring policy, measured continuously.
- **Off-policy**: measured **once every 10,000 steps** by running the deterministic policy
(or, for SAC, the **mean** policy) **without action noise** for **ten trajectories**, and
reporting the average return over those test trajectories.
## Reference Tables
Benchmark setup:
| Parameter | On-policy (VPG, TRPO, PPO) | Off-policy (DDPG, TD3, SAC) |
|-----------|---------------------------|----------------------------|
| Network architecture | (64, 32), tanh units | (256, 256), relu units |
| Batch | 4000 environment steps per batch update | Minibatches of 100 per gradient step |
| Performance metric | Average batch trajectory return per epoch | Deterministic/mean policy, no noise, 10 trajectories every 10k steps |
| Research-grade? | No — use Baselines for TRPO/PPO | Yes |
Common to all:
| Setting | Value |
|---------|-------|
| Environments | HalfCheetah-v3, Hopper-v3, Walker2d-v3, Swimmer-v3, Ant-v3 (MuJoCo Gym suite) |
| Timesteps | 3M |
| Random seeds | 10 per experiment |
| Plot content | Solid line = mean over seeds; shaded = std dev over seeds |
| Smoothing | Averaged over a window of 11 epochs |
| Other hyperparameters | Left at Spinning Up defaults; see each algorithm page |
Both PyTorch and TF1 versions were benchmarked in every environment, plus dedicated
head-to-head PyTorch-vs-TF1 pages for VPG, PPO, DDPG, TD3 and SAC. (TRPO has no PyTorch
implementation, so it has no head-to-head page.)
## Mental Models
- **Reporting mean and std dev over 10 seeds *as shaded bands* is the visual form of ch10's
"remove stochasticity as a confounder."** A benchmark plot without a seed band is telling
you less than it appears to.
- **The architecture split is itself a finding**: on-policy at (64, 32)/tanh and off-policy
at (256, 256)/relu is the convention these algorithm families settled into, not an
arbitrary choice, and copying the wrong family's defaults is a real source of bad results.
- **Honest self-assessment is a feature of the resource**, not a caveat. A benchmarks page
that names which of its own implementations you should not use is doing the reader's
ch10 rigor work for them.
## Anti-patterns
- **Using Spinning Up's PPO or TRPO as a paper baseline.** The page says explicitly to use
Baselines for those. Under-powered baselines are the ch10 failure of "handicapping the
baseline," even when it is unintentional.
- **Comparing an on-policy `AverageEpRet` against an off-policy `AverageTestEpRet`** as if
they were the same quantity. Use the plotter's `Performance` alias (ch6), which resolves
per family.
- **Reading a single-seed curve as a result.** These plots are 10 seeds precisely because
fewer is not informative.
## Key Takeaways
1. 3M timesteps, five MuJoCo environments, 10 seeds, both backends.
2. DDPG, TD3, SAC: research-usable. VPG, TRPO, PPO: educational only — use Baselines.
3. The weakness is attributed to two named omissions: observation normalization and
normalized value regression targets.
4. On-policy and off-policy performance are different measurements, not different numbers
of the same measurement.
5. Architecture defaults differ by family: (64,32)/tanh on-policy, (256,256)/relu off-policy.
## Connects To
- **Ch 1**: "reasonably good" — this chapter is the quantification of that phrase.
- **Ch 6**: the `Performance` alias, which exists because of the metric split described here.
- **Ch 10**: the seed and fair-baseline standards this benchmark actually meets.
- **Ch 14-19**: per-algorithm hyperparameter defaults referenced as "left at default settings."

93
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch14-vpg.md Normal file
View file

@ -0,0 +1,93 @@
# Chapter 14: Vanilla Policy Gradient (VPG)
## Core Idea
Push up the probabilities of actions that lead to higher return and push down the
probabilities of actions that lead to lower return, until you arrive at the optimal policy.
That is the whole idea underlying policy gradients, and VPG is its most direct expression.
## Quick Facts
- **On-policy.**
- Works with **discrete or continuous** action spaces.
- The Spinning Up implementation **supports MPI parallelization**.
## Key Equations
The advantage-weighted policy gradient (the end point of ch9):
```
grad_theta J(pi_theta) = E_{tau ~ pi_theta}[ sum_{t=0}^{T} grad_theta log pi_theta(a_t|s_t) * A^{pi_theta}(s_t,a_t) ]
```
Stochastic gradient ascent on policy performance:
```
theta_{k+1} = theta_k + alpha * grad_theta J(pi_theta_k)
```
**The notation mismatch worth knowing about:** `J(pi_theta)` here denotes the expected
**finite-horizon undiscounted** return, but policy gradient implementations typically compute
**advantage estimates based on the infinite-horizon discounted return** — despite otherwise
using the finite-horizon undiscounted policy gradient formula. This is the ch7 blurring of
the two return formulations, appearing in production code.
## Frameworks Introduced
- **The VPG loop** (the template TRPO and PPO both modify, so learn it once):
1. Collect a set of trajectories `D_k` by running `pi_k = pi(theta_k)` in the environment.
2. Compute rewards-to-go `Rhat_t`.
3. Compute advantage estimates `Ahat_t` (any method of advantage estimation) based on the
current value function `V_{phi_k}`.
4. Estimate the policy gradient:
`ghat_k = (1/|D_k|) * sum_{tau in D_k} sum_t grad_theta log pi_theta(a_t|s_t)|_{theta_k} * Ahat_t`
5. Update the policy by standard gradient ascent `theta_{k+1} = theta_k + alpha_k * ghat_k`,
or another gradient ascent algorithm like Adam.
6. Fit the value function by regression on mean-squared error:
`phi_{k+1} = argmin_phi (1/(|D_k| T)) * sum sum (V_phi(s_t) - Rhat_t)^2`, typically via
gradient descent.
## Exploration vs. Exploitation
VPG trains a **stochastic policy in an on-policy way**, so it explores by sampling actions
from the latest version of that policy. How random that is depends on both initial conditions
and the training procedure. **Over training the policy typically becomes progressively less
random**, because the update rule encourages exploiting rewards already found — **which may
cause the policy to get trapped in local optima.** (This paragraph is identical for TRPO and
PPO; it is a property of the on-policy stochastic-policy family, not of VPG specifically.)
## Code Examples
Loading and using a trained PyTorch model:
```python
ac = torch.load('path/to/model.pt')
actions = ac.act(torch.as_tensor(obs, dtype=torch.float32))
```
The TF1 saved graph exposes three keys: `x` (state input placeholder), `pi` (samples an
action conditioned on `x`), and `v` (value estimate for states in `x`).
## Anti-patterns
- **Expecting monotonic improvement.** VPG has none; the step size is unconstrained and a
bad step can collapse performance. That collapse is what TRPO (ch15) exists to prevent.
- **Under-fitting the value function.** Exercise 2.1 (ch12) shows a policy gradient agent
with an untrained value function gets stuck early. The advantage estimate is only as good
as `V_phi`.
- **Reading the loss.** See ch9's worked example.
## Reference Tables
| Relevant paper | Why it is on the list |
|----------------|----------------------|
| Sutton et al. 2000, *Policy Gradient Methods for RL with Function Approximation* | A timeless classic of RL theory; contains references to the earlier work that led to modern policy gradients |
| Schulman 2016(a), *Optimizing Expectations* | Chapter 2 is a lucid introduction to policy gradient theory, including pseudocode |
| Duan et al. 2016, *Benchmarking Deep RL for Continuous Control* | A clear benchmark paper showing how VPG in the deep RL setting compares with other deep RL algorithms |
| Schulman et al. 2016(b), *High Dimensional Continuous Control Using GAE* | Spinning Up's VPG uses GAE for computing the policy gradient |
## Key Takeaways
1. VPG is the advantage-weighted policy gradient plus mean-squared value regression — nothing else.
2. The six-step loop is the base template; TRPO changes step 5, PPO changes step 5 differently.
3. Discrete or continuous, MPI-parallelizable, both backends available.
4. Its weakness is step-size safety: nothing stops a single update from collapsing the policy.
5. Exploration decays as a side effect of exploitation, with local optima the known risk.
## Connects To
- **Ch 9**: the derivation that produces this exact gradient, and GAE.
- **Ch 15**: TRPO, which constrains the step this chapter leaves unconstrained.
- **Ch 12**: Exercise 2.1, on what happens when `V_phi` is not fit.
- **Ch 13**: the benchmark caveat — VPG is educational-grade, not research-grade.

122
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch15-trpo.md Normal file
View file

@ -0,0 +1,122 @@
# Chapter 15: Trust Region Policy Optimization (TRPO)
## Core Idea
Take the **largest step possible to improve performance** while satisfying a constraint on
how close the new and old policies are — measured in **KL-divergence between policies**, not
distance in parameter space, because seemingly small parameter differences can produce very
large performance differences.
## Quick Facts
- **On-policy.**
- Works with **discrete or continuous** action spaces.
- **Supports MPI parallelization.**
- **Tensorflow only** in Spinning Up; there is no PyTorch TRPO.
## Key Equations
The theoretical update — maximize surrogate advantage subject to a KL trust region:
```
theta_{k+1} = argmax_theta L(theta_k, theta)
s.t. Dbar_KL(theta || theta_k) <= delta
```
The **surrogate advantage** measures how `pi_theta` performs relative to the old policy
*using data from the old policy* (an importance-weighted advantage):
```
L(theta_k, theta) = E_{s,a ~ pi_theta_k}[ (pi_theta(a|s) / pi_theta_k(a|s)) * A^{pi_theta_k}(s,a) ]
Dbar_KL(theta || theta_k) = E_{s ~ pi_theta_k}[ D_KL( pi_theta(.|s) || pi_theta_k(.|s) ) ]
```
Both the objective and the constraint are **zero at `theta = theta_k`**, and the gradient of
the constraint is **also zero there**.
## Frameworks Introduced
- **Taylor-expand to make it solvable.** Expand objective and constraint to leading order
around `theta_k`:
```
L(theta_k, theta) ~= g^T (theta - theta_k)
Dbar_KL(theta||theta_k) ~= 0.5 * (theta - theta_k)^T H (theta - theta_k)
```
**By happy coincidence, `g` — the gradient of the surrogate advantage at `theta_k` — is
exactly equal to the policy gradient `grad_theta J(pi_theta)`.** Lagrangian duality then
solves the approximate problem analytically:
```
theta_{k+1} = theta_k + sqrt( 2*delta / (g^T H^{-1} g) ) * H^{-1} g
```
**Stopping here would be exactly the Natural Policy Gradient.**
- **Backtracking line search — the fix that makes it TRPO.** Because the Taylor expansion
introduces approximation error, that update may violate the KL constraint or fail to improve
the surrogate advantage. So:
```
theta_{k+1} = theta_k + alpha^j * sqrt( 2*delta / (g^T H^{-1} g) ) * H^{-1} g
```
where `alpha` in (0,1) is the **backtracking coefficient** and `j` is the **smallest
nonnegative integer** such that the new policy satisfies the KL constraint and produces a
positive surrogate advantage.
- When to use this pattern generally: whenever you solve an approximated problem and can
cheaply check the exact condition, shrink the step until the exact condition holds.
- **Conjugate gradient instead of a matrix inverse.** Computing and storing `H^{-1}` is
painfully expensive for networks with thousands or millions of parameters. TRPO solves
`Hx = g` for `x = H^{-1} g` with conjugate gradient, which needs only a function computing
the matrix-vector product `Hx`, never `H` itself:
```
Hx = grad_theta( (grad_theta Dbar_KL(theta || theta_k))^T x )
```
## Reference Tables
| Hyperparameter | Role |
|----------------|------|
| `delta` | KL-divergence limit — the size of the trust region |
| `alpha` | Backtracking coefficient, in (0,1) |
| `K` | Maximum number of backtracking steps; `j` ranges over {0, 1, ..., K} |
The full loop is VPG's six steps with steps 5 replaced by: (a) conjugate gradient to compute
`xhat_k ~= Hhat_k^{-1} ghat_k`, where `Hhat_k` is the Hessian of the sample average
KL-divergence, then (b) the backtracking line search using
`theta_{k+1} = theta_k + alpha^j * sqrt(2 delta / (xhat_k^T Hhat_k xhat_k)) * xhat_k`.
## Mental Models
- **Trust region in policy space, not parameter space.** This is the whole insight. Ordinary
policy gradient methods keep policies close in *parameter* space, but small parameter
differences can mean very large performance differences — **so a single bad step can
collapse the policy performance.** That is what makes large step sizes dangerous with
vanilla policy gradients, and it is why VPG's sample efficiency suffers: it has to take
small steps.
- **TRPO buys back sample efficiency by making large steps safe**, and tends to improve
performance **quickly and monotonically**.
- Read the algorithm as **three nested approximations** — surrogate objective, Taylor
expansion, conjugate gradient — each with a guard, the last being the line search.
## Exploration vs. Exploitation
Identical to VPG: stochastic policy, on-policy sampling, randomness decaying over training as
the update rule pushes toward exploiting known rewards, with local optima as the risk.
## Anti-patterns
- **Skipping the line search** and shipping the analytic solution. That is Natural Policy
Gradient, and the approximation error it inherits may violate the constraint or fail to
improve anything.
- **Forming `H` explicitly.** The matrix-vector product formulation exists because the matrix
does not fit.
- **Reaching for TRPO in PyTorch inside Spinning Up.** It does not exist; use PPO.
## Key Takeaways
1. Constrain the *policy* change in KL, not the *parameter* change — the core contribution.
2. Surrogate advantage is importance-weighted old-policy data; both it and the KL constraint
vanish at `theta_k`.
3. `g` (surrogate advantage gradient) equals the policy gradient — that is why VPG's machinery
is reusable here.
4. Analytic solution = Natural Policy Gradient; TRPO adds a backtracking line search on top.
5. Conjugate gradient replaces the intractable `H^{-1}`.
6. Three relevant papers: Schulman 2015 (original), Schulman 2016 (GAE, used here), Kakade and
Langford 2002 (the theory motivating and connecting to TRPO's foundations).
## Connects To
- **Ch 14**: the VPG loop TRPO modifies, and the collapse risk it removes.
- **Ch 16**: PPO, which pursues the same goal with first-order methods.
- **Ch 12**: Exercise 2.1 uses TRPO to demonstrate the value-fitting failure mode.
- **Ch 10**: "monotonic improvement theory" — the optional math background this rests on.

123
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch16-ppo.md Normal file
View file

@ -0,0 +1,123 @@
# Chapter 16: Proximal Policy Optimization (PPO)
## Core Idea
PPO answers the same question as TRPO — how do you take the biggest possible improvement
step without stepping so far that you cause performance collapse — but with **first-order
methods and a few tricks** instead of a complex second-order method. It is significantly
simpler to implement and empirically seems to perform at least as well.
## Quick Facts
- **On-policy.**
- Works with **discrete or continuous** action spaces.
- **Supports MPI parallelization.**
- Two variants exist; Spinning Up implements and documents **PPO-Clip**, the primary variant
used at OpenAI.
## Frameworks Introduced
- **The two PPO variants:**
- **PPO-Penalty** approximately solves a KL-constrained update like TRPO, but **penalizes
the KL-divergence in the objective** instead of making it a hard constraint, and
**automatically adjusts the penalty coefficient** over training so it stays appropriately
scaled.
- **PPO-Clip** has **no KL term in the objective and no constraint at all.** It relies on
specialized **clipping in the objective function** to remove the incentive for the new
policy to get far from the old.
- **The PPO-Clip objective.** Update by
`theta_{k+1} = argmax_theta E_{s,a ~ pi_theta_k}[ L(s,a,theta_k,theta) ]`, typically with
**multiple steps of (usually minibatch) SGD**. The published form:
```
L = min( ratio * A, clip(ratio, 1-eps, 1+eps) * A )
where ratio = pi_theta(a|s) / pi_theta_k(a|s)
```
and `eps` is a small hyperparameter roughly saying how far the new policy may go.
- **The simplified equivalent form** (this is the version Spinning Up actually implements,
and it is much easier to reason about):
```
L = min( ratio * A, g(eps, A) )
where g(eps, A) = (1 + eps) * A if A >= 0
(1 - eps) * A if A < 0
```
## Worked Example
**Why clipping removes the incentive to move far — the two cases.**
**Advantage positive.** The term reduces to `min(ratio, 1+eps) * A`. Because `A > 0`, the
objective increases as the action becomes more likely, i.e. as `pi_theta(a|s)` increases.
But the `min` caps how much. Once `pi_theta(a|s) > (1+eps) * pi_theta_k(a|s)`, the min kicks
in and the term hits a ceiling of `(1+eps) * A`. **The new policy does not benefit by going
far away from the old policy.**
**Advantage negative.** The term reduces to `max(ratio, 1-eps) * A`. Because `A < 0`, the
objective increases as the action becomes *less* likely, i.e. as `pi_theta(a|s)` decreases.
The `max` caps how much: once `pi_theta(a|s) < (1-eps) * pi_theta_k(a|s)`, it hits a ceiling
of `(1-eps) * A`. **Again: the new policy does not benefit by going far away from the old.**
So **clipping serves as a regularizer by removing incentives for the policy to change
dramatically**, and `eps` corresponds to how far the new policy can go while still profiting
from the objective.
**The honest caveat, and Spinning Up's answer.** Clipping goes a long way toward reasonable
updates, but **it is still possible to end up with a new policy too far from the old**, and
different PPO implementations use a variety of tricks to stave this off. Spinning Up uses a
particularly simple one: **early stopping — if the mean KL-divergence of the new policy from
the old grows beyond a threshold, stop taking gradient steps.** Note that this reintroduces a
KL measurement into an algorithm defined by not having one; the clip is the incentive
mechanism, the KL check is the safety net.
## Reference Tables
The PPO-Clip loop, as a diff against VPG's six steps (ch14):
| Step | VPG | PPO-Clip |
|------|-----|----------|
| 1-3 | Collect trajectories, rewards-to-go, advantage estimates | identical |
| 4-5 | Estimate `ghat_k`, single ascent step | **Maximize the PPO-Clip objective, typically via multiple steps of stochastic gradient ascent with Adam** |
| 6 | Fit `V_phi` by MSE regression on `Rhat_t` | identical |
The key structural difference from VPG and TRPO: **multiple gradient steps per batch of data**,
which is where the sample-efficiency gain comes from and why the policy needs restraining at all.
| Relevant paper | Why it is on the list |
|----------------|----------------------|
| Schulman et al. 2017, *Proximal Policy Optimization Algorithms* | The original PPO paper |
| Schulman et al. 2016, *High Dimensional Continuous Control Using GAE* | Spinning Up's PPO uses GAE for the policy gradient |
| Heess et al. 2017, *Emergence of Locomotion Behaviours in Rich Environments* | Large-scale empirical analysis of behaviors learned by PPO agents in complex environments — though it uses PPO-Penalty, not PPO-Clip |
## Exploration vs. Exploitation
Identical to VPG and TRPO: stochastic policy, on-policy sampling, decaying randomness, local
optima as the failure mode.
## Mental Models
- **PPO trades a hard guarantee for implementability.** TRPO enforces the trust region;
PPO removes the *incentive* to leave it. That is a weaker statement, and PPO's practical
dominance says the weaker statement is usually enough.
- **The clip is one-sided per case.** It only ever caps the *beneficial* direction. It never
penalizes moving the wrong way — it just stops paying you for moving further the right way.
- Careful with terminology: **ch9 notes PPO is often called a policy gradient algorithm,
though this is slightly inaccurate** — it maximizes a surrogate objective rather than
following `grad J` directly.
## Anti-patterns
- **Implementing PPO-Clip and calling the result PPO without saying which variant.** Two
public implementations named "PPO" (ModularRL, rllab) implement PPO-**Penalty**; comparing
against them without noticing is a real reproduction hazard the docs explicitly flag.
- **Relying on the clip alone at large step counts.** More SGD steps per batch means more
opportunity to drift; that is precisely why the early-stopping KL check exists.
- **Tuning `eps` as if it were a learning rate.** It bounds policy change per update; the
number of SGD steps and the learning rate interact with it.
## Key Takeaways
1. Same goal as TRPO, first-order machinery: simpler to implement, at least as good empirically.
2. PPO-Clip (no KL term, no constraint) is the OpenAI default and Spinning Up's implementation.
3. The simplified `g(eps, A)` form makes the mechanism obvious: a ceiling in the profitable
direction, in both the positive- and negative-advantage cases.
4. Clipping is an incentive change, not a guarantee — Spinning Up backs it with KL early stopping.
5. Multiple minibatch SGD steps per batch is the structural difference from VPG and TRPO.
## Connects To
- **Ch 15**: TRPO's hard constraint, the thing PPO replaces.
- **Ch 9**: the surrogate-objective framing and why "policy gradient algorithm" is imprecise here.
- **Ch 12**: Exercise 1.2 asks you to implement the MLP diagonal Gaussian policy for PPO.
- **Ch 13**: use OpenAI Baselines' PPO, not this one, for research comparisons.

122
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch17-ddpg.md Normal file
View file

@ -0,0 +1,122 @@
# Chapter 17: Deep Deterministic Policy Gradient (DDPG)
## Core Idea
DDPG concurrently learns a Q-function and a policy: it uses off-policy data and the Bellman
equation to learn the Q-function, and uses the Q-function to learn the policy. It exists
because `max_a Q*(s,a)` — trivial over discrete actions — is intractable over continuous ones,
and DDPG replaces that max with a learned, differentiable approximation.
## Quick Facts
- **Off-policy.**
- **Continuous action spaces only.**
- Can be thought of as **deep Q-learning for continuous action spaces.**
- The Spinning Up implementation **does not support parallelization.**
## Frameworks Introduced
- **The continuous-argmax problem, and the substitution that solves it.** With finitely many
discrete actions, the max poses no problem — compute Q for each and compare, which also
immediately hands you the maximizing action. With a continuous action space you cannot
exhaustively evaluate, and a general optimization subroutine would have to run **every time
the agent wants to take an action**, which is unacceptable. But because the action space is
continuous, `Q*(s,a)` is presumed **differentiable with respect to the action** — so you can
learn a policy `mu(s)` by gradient ascent and approximate
`max_a Q(s,a) ~= Q(s, mu(s))`.
- When to use this pattern: any time an inner argmax over a continuous variable sits inside
an outer loop. Amortize it into a learned function.
- **Mean-Squared Bellman Error (MSBE)** — the loss that all deep Q-learning minimizes:
```
L(phi, D) = E_{(s,a,r,s',d) ~ D}[ ( Q_phi(s,a) - ( r + gamma*(1-d)*max_a' Q_phi(s',a') ) )^2 ]
```
`d` indicates whether `s'` is terminal, using the Python convention of `True` as 1: when
`d == True` the Q-function should show the agent gets no additional reward after this state.
- **Trick One: Replay Buffers.** All standard algorithms training a deep network to approximate
`Q*(s,a)` use an experience replay buffer — the set `D` of previous experiences.
- **The buffer size is a real tuning decision, in both directions:** it should be large
enough to contain a wide range of experiences, but keeping everything may not be good
either. **Use only the very-most recent data and you overfit to it and things break; use
too much experience and you may slow down learning.**
- **Why old data is admissible at all** — this is the cleanest statement of what off-policy
means: **the Bellman equation does not care which transition tuples are used, how the
actions were selected, or what happens after a given transition**, because the optimal
Q-function must satisfy it for *all* possible transitions. So any transition you have ever
experienced is fair game for MSBE minimization.
- **Trick Two: Target Networks.** The target `r + gamma*(1-d)*max_a' Q_phi(s',a')` depends on
the same parameters `phi` being trained, which makes MSBE minimization unstable. The fix is
a second network that lags the first, with parameters `phi_targ`.
- **DQN-style**: copy the main network into the target every fixed number of steps.
- **DDPG-style**: **polyak averaging**, once per main network update:
`phi_targ <- rho * phi_targ + (1-rho) * phi`, with `rho` in (0,1), usually close to 1.
(`rho` is called `polyak` in the code.)
- **DDPG detail: the max in the target.** Since the max over continuous actions is the original
problem, DDPG uses a **target policy network** `mu_theta_targ` to compute an action that
approximately maximizes `Q_phi_targ`, found the same way as the target Q-function — by
polyak averaging the policy parameters over training. Final Q-loss:
```
L(phi,D) = E[ ( Q_phi(s,a) - ( r + gamma*(1-d)*Q_phi_targ(s', mu_theta_targ(s')) ) )^2 ]
```
- **Policy learning is one line**: maximize `E_{s~D}[ Q_phi(s, mu_theta(s)) ]` by gradient
ascent **with respect to policy parameters only** — the Q-function parameters are treated
as constants.
## Exploration vs. Exploitation
DDPG trains a **deterministic policy off-policy**. Because the policy is deterministic,
on-policy exploration would probably not try a wide enough variety of actions early on to
find useful learning signal. So **noise is added to actions at training time**.
- The original DDPG paper recommended **time-correlated OU noise**, but **more recent results
suggest uncorrelated, mean-zero Gaussian noise works perfectly well** — and since it is
simpler, it is preferred.
- You may reduce the noise scale over training to get higher-quality data. **Spinning Up does
not do this and keeps the noise scale fixed throughout.**
- **At test time, no noise is added** — that is how you see how well the policy exploits what
it has learned.
- **`start_steps` trick**: for a fixed number of steps at the beginning, the agent takes
actions sampled from a **uniform random distribution over valid actions**, then reverts to
normal DDPG exploration.
## Reference Tables
The DDPG loop:
| Step | Action |
|------|--------|
| Init | Set target parameters equal to main: `theta_targ <- theta`, `phi_targ <- phi` |
| Act | Observe `s`, select `a = clip(mu_theta(s) + epsilon, a_Low, a_High)`, `epsilon ~ N` |
| Store | Execute `a`, observe `s'`, `r`, `d`; store `(s,a,r,s',d)` in `D`; reset if terminal |
| Update | Sample a batch `B` from `D`; compute targets `y = r + gamma*(1-d)*Q_phi_targ(s', mu_theta_targ(s'))` |
| | One gradient descent step on `(Q_phi(s,a) - y)^2` averaged over `B` |
| | One gradient ascent step on `Q_phi(s, mu_theta(s))` averaged over `B` |
| | Polyak-update both target networks |
| Relevant paper | Why it is on the list |
|----------------|----------------------|
| Silver et al. 2014, *Deterministic Policy Gradient Algorithms* | Establishes the theory underlying deterministic policy gradients (DPG) |
| Lillicrap et al. 2016, *Continuous Control With Deep RL* | Adapts the theoretically-grounded DPG algorithm to the deep RL setting, giving DDPG |
## Anti-patterns
- **DDPG on a discrete action space.** The entire design is the continuous-action workaround.
- **Trusting the original paper's full recipe.** Ch10 uses DDPG as its example: the paper
suggests a complex architecture, an initialization scheme and batch normalization, none of
which are strictly necessary, and some of the best DDPG results use simpler networks.
- **Assuming DDPG is stable.** It is **frequently brittle with respect to hyperparameters and
other kinds of tuning** — that is the premise of ch18.
- **Shape bugs in the Q-function.** Exercise 2.2 (ch12) plants exactly one in DDPG's critic
and it silently ruins learning through broadcasting.
## Key Takeaways
1. `Q(s, mu(s))` replaces `max_a Q(s,a)` — the whole reason DDPG exists.
2. Replay buffers are licensed by the Bellman equation's indifference to how data was collected.
3. Target networks fix the moving-target instability; DDPG uses polyak averaging, DQN uses copies.
4. Two target networks in DDPG: target Q *and* target policy.
5. Explore with additive Gaussian action noise plus uniform-random `start_steps`; no noise at test.
6. Replay buffer size is a genuine trade-off, wrong in both directions.
## Connects To
- **Ch 7**: `a*(s) = argmax_a Q*(s,a)` and the Bellman optimality equation this is built on.
- **Ch 8**: DDPG as the named interpolation between policy optimization and Q-learning.
- **Ch 18**: TD3, the three tricks that fix DDPG's brittleness.
- **Ch 12**: Exercise 2.2, the silent bug planted in this algorithm.

107
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch18-td3.md Normal file
View file

@ -0,0 +1,107 @@
# Chapter 18: Twin Delayed DDPG (TD3)
## Core Idea
DDPG's common failure mode is that **the learned Q-function begins to dramatically
overestimate Q-values, which then breaks the policy, because the policy exploits the errors
in the Q-function.** TD3 fixes this with three tricks, and nothing else changes.
## Quick Facts
- **Off-policy.**
- **Continuous action spaces only.**
- The Spinning Up implementation **does not support parallelization.**
## Frameworks Introduced
- **Trick One: Clipped Double-Q Learning.** Learn **two** Q-functions instead of one (hence
"twin") and **use the smaller of the two Q-values to form the targets** in the Bellman error
loss functions. Both are then regressed toward that single shared target:
```
y(r,s',d) = r + gamma*(1-d) * min_{i=1,2} Q_{phi_i,targ}(s', a'(s'))
L(phi_1, D) = E[ (Q_phi_1(s,a) - y)^2 ]
L(phi_2, D) = E[ (Q_phi_2(s,a) - y)^2 ]
```
**Using the smaller Q-value for the target, and regressing toward that, helps fend off
overestimation** — a pessimistic estimate cannot be exploited the way an optimistic one can.
- **Trick Two: "Delayed" Policy Updates.** Update the policy (and the target networks) **less
frequently than the Q-function**. **The paper recommends one policy update for every two
Q-function updates.** This **damps the volatility that normally arises in DDPG because of
how a policy update changes the target.** Controlled by `policy_delay`.
- **Trick Three: Target Policy Smoothing.** Add **clipped** noise to each dimension of the
target action, then clip the result back into the valid action range:
```
a'(s') = clip( mu_theta_targ(s') + clip(epsilon, -c, c), a_Low, a_High ), epsilon ~ N(0, sigma)
```
**This is a regularizer.** It addresses a specific DDPG failure: **if the Q-function
approximator develops an incorrect sharp peak for some actions, the policy will quickly
exploit that peak and then have brittle or incorrect behavior.** Smoothing Q over similar
actions averts it.
- **Policy learning is unchanged from DDPG except in which critic it uses:** maximize
`E_{s~D}[ Q_phi_1(s, mu_theta(s)) ]` — **just the first Q-function.** (SAC differs here; see
ch19.)
## Reference Tables
| Trick | Fixes | Mechanism |
|-------|-------|-----------|
| Clipped double-Q | Q-value overestimation | Two critics, `min` of the two forms the shared target |
| Delayed policy updates | Volatility from policy updates moving the target | One policy + target update per `policy_delay` critic updates |
| Target policy smoothing | Policy exploiting sharp incorrect peaks in Q | Clipped Gaussian noise on the target action, then clip to action bounds |
| Hyperparameter | Role |
|----------------|------|
| `sigma` | Std dev of the target-smoothing noise |
| `c` | Clip bound on that noise |
| `policy_delay` | Q-updates per policy update; paper recommends 2 |
| `rho` (`polyak`) | Target network averaging coefficient, inherited from DDPG |
| `start_steps` | Initial uniform-random action steps, inherited from DDPG |
The loop is DDPG's, with two changes inside the update block: compute the smoothed target
action `a'(s')` first, update **both** Q-functions toward the clipped-double-Q target, and
then — **only when `j mod policy_delay == 0`** — take the policy ascent step and polyak-update
all three target networks.
## Exploration vs. Exploitation
Identical to DDPG: deterministic policy explored with **uncorrelated mean-zero Gaussian action
noise** at training time, optionally decayed (Spinning Up keeps it fixed), **no noise at test
time**, and a `start_steps` phase of uniform random actions at the beginning.
Note the distinction that is easy to blur: **exploration noise is added to the acting policy;
target policy smoothing noise is added to the target action inside the update.** Two different
noises, two different jobs.
## Mental Models
- **All three tricks are forms of pessimism or patience.** Take the smaller estimate, wait
longer before acting on the critic, and refuse to believe a Q-value that does not hold up
under a small action perturbation. Overestimation is the disease; conservatism is the cure.
- **Think of the policy as an adversary against your critic's errors.** Anything the critic
gets wrong in the optimistic direction is exactly what the policy will find. This is the
same failure shape as model bias in ch8's model-based methods.
- **"Together, these three tricks result in substantially improved performance over baseline
DDPG"** — they are presented as a package, and the exercise (ch12, 1.3) has you implement
the losses that contain all three.
## Anti-patterns
- **Implementing only clipped double-Q and calling it TD3.** The three tricks address three
distinct failure modes and the paper's result is for the package.
- **Using `min` over the two critics for the *policy* loss.** TD3 uses `Q_phi_1` alone for the
policy; using the min there is a SAC choice, not a TD3 one.
- **Updating the target networks on every critic step.** The target updates are inside the
`policy_delay` branch, along with the policy update.
## Key Takeaways
1. TD3 = DDPG + clipped double-Q + delayed policy updates + target policy smoothing.
2. The disease being treated is Q-value overestimation, which the policy actively exploits.
3. `min` of two critics forms the target; the policy maximizes `Q_phi_1` only.
4. Policy and target-network updates both live behind `policy_delay` (recommended 2).
5. Target smoothing noise is clipped twice: clip the noise to `[-c, c]`, then clip the
resulting action to the valid range.
6. Relevant paper: Fujimoto et al, 2018, *Addressing Function Approximation Error in
Actor-Critic Methods.*
## Connects To
- **Ch 17**: DDPG, whose brittleness is TD3's entire premise.
- **Ch 19**: SAC, which borrows clipped double-Q and gets smoothing for free from stochasticity.
- **Ch 12**: Exercise 1.3 asks you to write exactly these losses.
- **Ch 13**: TD3 is one of the three research-grade Spinning Up implementations.

133
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch19-sac.md Normal file
View file

@ -0,0 +1,133 @@
# Chapter 19: Soft Actor-Critic (SAC)
## Core Idea
SAC optimizes a **stochastic** policy in an **off-policy** way, forming a bridge between
stochastic policy optimization and DDPG-style approaches. Its central feature is **entropy
regularization**: the policy is trained to maximize a trade-off between expected return and
the randomness of the policy.
## Quick Facts
- **Off-policy.**
- The version implemented here is **continuous action spaces only** — an alternate version,
with a slightly changed policy update rule, can handle discrete action spaces.
- **Does not support parallelization.**
- SAC is **not a direct successor to TD3** — they were published roughly concurrently — but it
incorporates the clipped double-Q trick, and its inherent policy stochasticity gives it
something like target policy smoothing for free.
## Frameworks Introduced
- **Entropy-regularized RL.** Entropy `H(P) = E_{x~P}[-log P(x)]` says how random a random
variable is: a coin that almost always comes up heads has low entropy, a fair coin has high
entropy. The agent gets a bonus reward at each timestep proportional to policy entropy,
changing the RL problem to:
```
pi* = argmax_pi E_{tau~pi}[ sum_t gamma^t ( R(s_t,a_t,s_{t+1}) + alpha * H(pi(.|s_t)) ) ]
```
`alpha > 0` is the trade-off coefficient. **This has a close connection to the
exploration-exploitation trade-off: increasing entropy results in more exploration, which
can accelerate learning later on, and can prevent the policy from prematurely converging to
a bad local optimum.**
- **The modified value functions.** `V^pi` includes the entropy bonus from **every** timestep;
`Q^pi` includes it from every timestep **except the first**. They connect by
```
V^pi(s) = E_{a~pi}[ Q^pi(s,a) ] + alpha * H(pi(.|s))
= E_{a~pi}[ Q^pi(s,a) - alpha * log pi(a|s) ]
```
**This setup is a little bit arbitrary** — you could instead have `Q^pi` include the first
timestep's bonus — and **the choice of definition varies slightly across papers on the
subject.** Check before comparing equations across sources.
- **What SAC keeps from TD3, and what it changes.** SAC learns a policy `pi_theta` and two
Q-functions `Q_phi_1`, `Q_phi_2`.
- **Same as TD3**: both Q-functions learned by MSBE minimization regressing to a **single
shared target**; the target computed using **target Q-networks obtained by polyak
averaging**; the **clipped double-Q trick**.
- **Different from TD3**: (1) the target includes an **entropy regularization term**;
(2) the **next-state actions in the target come from the current policy, not a target
policy**; (3) **no explicit target policy smoothing** — TD3 trains a deterministic policy
and so needs added noise to smooth, whereas **SAC's policy stochasticity is sufficient to
get a similar effect**.
- **The Q-loss.** Rewrite the entropy-regularized Bellman equation using
`H = -log pi`, approximate the expectation with samples, and take the min over the two
critics:
```
y(r,s',d) = r + gamma*(1-d) * ( min_{j=1,2} Q_{phi_j,targ}(s', a'~) - alpha * log pi_theta(a'~|s') )
where a'~ ~ pi_theta(.|s')
```
**Notation matters here:** `r` and `s'` come from the **replay buffer**, but `a'~` must be
**sampled fresh from the current policy** — hence the tilde. Mixing these up is a silent bug.
- **Policy learning via the reparameterization trick.** The policy should maximize `V^pi(s)`,
i.e. `E_{a~pi}[ Q^pi(s,a) - alpha log pi(a|s) ]`. The expectation's distribution depends on
the policy parameters, which is the pain point; the reparameterization trick rewrites it as
an expectation over **noise**, which has no parameter dependence. Using a **squashed Gaussian
policy**:
```
a~_theta(s, xi) = tanh( mu_theta(s) + sigma_theta(s) * xi ), xi ~ N(0, I)
```
giving the policy objective
```
max_theta E_{s~D, xi~N}[ min_{j=1,2} Q_phi_j(s, a~_theta(s,xi)) - alpha * log pi_theta(a~_theta(s,xi)|s) ]
```
**almost the same as DDPG and TD3 policy optimization, except for the min-double-Q trick,
the stochasticity, and the entropy term.** Note that unlike TD3 (which uses `Q_phi_1` only),
**SAC uses the min of the two approximators in the policy loss too.**
## Reference Tables
**Two ways SAC's policy differs from the VPG/TRPO/PPO policies** — both are load-bearing:
| Difference | SAC | VPG / TRPO / PPO |
|-----------|-----|------------------|
| **Squashing function** | `tanh` ensures actions are bounded to a finite range. Before the tanh the policy is a factored Gaussian; after it, it is not. Log-probabilities are still computable in closed form (see the paper appendix) | No squashing |
| **Std dev parameterization** | log stds are **outputs of the neural network**, so they depend on state in a complex way | log stds are **state-independent parameter vectors** |
**SAC with state-independent log std devs, in the authors' experience, did not work.**
| Variant | Status |
|---------|--------|
| Fixed entropy coefficient `alpha` | What Spinning Up implements, for simplicity |
| Entropy-constrained (varies `alpha` over training) | **Generally preferred by practitioners** |
| Older SAC that also learns a value function `V_psi` | Superseded; the docs cover the modern version that omits it |
## Exploration vs. Exploitation
SAC trains a stochastic policy with entropy regularization and explores **on-policy**.
**`alpha` explicitly controls the explore-exploit trade-off** — higher means more exploration,
lower means more exploitation. **The right coefficient may vary from environment to environment
and could require careful tuning.** At test time, **remove the stochasticity and use the mean
action instead of a sample**; this tends to improve performance over the stochastic policy.
(This is why `test_policy` has a SAC-only `--deterministic` flag — see ch5.) `start_steps`
uniform-random exploration at the beginning applies here too.
## Anti-patterns
- **Evaluating SAC with the stochastic policy.** The correct evaluation policy is the
deterministic mean; `test_policy` defaults to the stochastic one and needs `-d`.
- **Using state-independent log stds in SAC** because that is what PPO does. It did not work.
- **Sampling the target's next action from the replay buffer.** It must come fresh from the
current policy.
- **Assuming SAC's `Q^pi` definition matches another paper's.** The placement of the
first-timestep entropy bonus is a convention that varies.
- **Treating `alpha` as a set-and-forget constant across environments.** It is the
explore-exploit dial and is environment-specific; the entropy-constrained variant exists
because tuning it by hand is hard.
## Key Takeaways
1. Entropy regularization is the defining feature: maximize return plus `alpha` times policy
entropy, which directly controls exploration.
2. SAC keeps TD3's clipped double-Q and polyak targets, adds an entropy term to the target,
uses the current policy (not a target policy) for next-state actions, and needs no explicit
smoothing.
3. The squashed Gaussian plus reparameterization trick makes the stochastic policy
differentiable end to end.
4. SAC uses `min` over both critics in the policy loss; TD3 uses only `Q_phi_1`.
5. State-dependent log stds are required — state-independent ones did not work.
6. Evaluate with the mean action. Spinning Up ships the fixed-`alpha` variant; practitioners
generally prefer the entropy-constrained one.
## Connects To
- **Ch 18**: TD3, whose clipped double-Q trick SAC borrows and whose smoothing it obviates.
- **Ch 7**: the diagonal Gaussian policy and the two ways of parameterizing log stds.
- **Ch 5**: the SAC-only `--deterministic` flag on `test_policy`.
- **Ch 10**: the reparameterization trick is on the required deep-learning background list.

99
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/chapters/ch20-logger-and-utilities.md Normal file
View file

@ -0,0 +1,99 @@
# Chapter 20: Logger, MPI Tools and Run Utils
## Core Idea
The three utility modules are the only code shared between algorithms — logging (which is
also model saving and config recording), MPI data-parallelism, and ExperimentGrid — and
each has a small fixed usage pattern worth memorizing.
## Frameworks Introduced
- **`Logger` and `EpochLogger`.** `Logger` carries the basic functionality: saving
diagnostics, hyperparameter configurations, the state of a training run, and the trained
model. `EpochLogger` adds a thin layer that makes it easy to track the **average, standard
deviation, min and max** of a diagnostic over each epoch **and across MPI workers**.
**All Spinning Up algorithm implementations use an EpochLogger.**
- The three-call pattern: **`store`** accumulates values into internal state,
**`log_tabular`** computes the statistics over everything stored, **`dump_tabular`**
writes to file and stdout. **The internal state is wiped clean after `log_tabular`,
to prevent leakage into the next epoch's statistics.**
- **The MPI + PyTorch pattern** — three steps, in this order:
1. At the beginning of the training script, call **`setup_pytorch_for_mpi()`**. This is the
fix for a real problem: each separate process's PyTorch instance tries to grab too many
threads and they clobber each other.
2. After constructing a PyTorch module, call **`sync_params(module)`**.
3. During gradient descent, call **`mpi_avg_grads`** after the backward pass and before the
optimizer step.
- The two main ingredients are therefore **syncing parameters** and **averaging gradients
before they are used by the adaptive optimizer** — the order matters, because averaging
after the optimizer step would give each worker a different adaptive state.
- **ExperimentGrid** — a tool for hyperparameter ablations, based on but simpler than rllab's
VariantGenerator. `eg.add(param_name, values, shorthand, in_name)` then
`eg.run(thunk, **run_kwargs)`; `ExperimentGrid.run` uses `call_experiment` to launch the
thunk. See ch4 for the full usage.
## Code Examples
The EpochLogger statistic pattern:
```python
from spinup.utils.logx import EpochLogger
epoch_logger = EpochLogger()
for i in range(10):
epoch_logger.store(Test=i)
epoch_logger.log_tabular('Test', with_min_and_max=True)
epoch_logger.dump_tabular()
```
```
-------------------------------------
| AverageTest | 4.5 |
| StdTest | 2.87 |
| MaxTest | 9 |
| MinTest | 0 |
-------------------------------------
```
The MPI gradient-averaging step, in place:
```python
optimizer.zero_grad()
loss = compute_loss(module)
loss.backward()
mpi_avg_grads(module) # averages gradient buffers across MPI processes
optimizer.step()
```
## Reference Tables
| Module | Contents |
|--------|----------|
| `spinup.utils.logx` | `Logger`, `EpochLogger` |
| `spinup.utils.mpi_tools` | Core MPI utilities |
| `spinup.utils.mpi_pytorch` | `setup_pytorch_for_mpi`, `sync_params`, `mpi_avg_grads` |
| `spinup.utils.mpi_tf` | AdamOptimizer across MPI processes. **Explicitly "a bit hacky"** — for something more sophisticated and general-purpose, the docs point to horovod |
| `spinup.utils.run_utils` | `ExperimentGrid`, `call_experiment`, `setup_logger_kwargs` |
| `spinup.utils.plot` | The plotter (documented in ch6) |
## Anti-patterns
- **Calling `mpi_avg_grads` after `optimizer.step()`.** The averaging must happen before the
adaptive optimizer consumes the gradients, or workers diverge in optimizer state.
- **Skipping `setup_pytorch_for_mpi()`.** The thread-clobbering problem it fixes is a real
performance failure, not a theoretical one.
- **Expecting `log_tabular` to be idempotent.** It wipes the accumulated state; calling it
twice for the same key in one epoch gives you statistics over nothing.
- **Reaching for `mpi_tf` as a general distributed-training solution.** The docs say to use
horovod instead if you need something serious.
## Key Takeaways
1. `store` / `log_tabular` / `dump_tabular` is the whole logger interface, and the state
resets on `log_tabular`.
2. `EpochLogger` aggregates across MPI workers as well as across an epoch.
3. MPI PyTorch is three calls in a fixed order: setup, sync params, average grads before step.
4. The logger is also the model-saving and config-recording path (ch5).
5. `mpi_tf` is acknowledged as hacky; horovod is the recommended alternative.
## Connects To
- **Ch 4**: ExperimentGrid usage and the CLI it backs.
- **Ch 5**: the outputs the logger writes — `progress.txt`, `config.json`, the save directories.
- **Ch 6**: the plotter, which reads what `dump_tabular` writes.
- **Ch 2**: OpenMPI, the system dependency all of this rests on.

87
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/cheatsheet.md Normal file
View file

@ -0,0 +1,87 @@
# Cheatsheet
Decision rules, thresholds and tells from Spinning Up. One page; keep it beside you.
## Which algorithm?
| If | Then | Because |
|----|------|---------|
| Discrete actions | PPO (or VPG to learn) | DDPG, TD3 and shipped SAC are continuous-only |
| Continuous, stability matters most | PPO | Directly optimizes performance; on-policy is stable |
| Continuous, sample efficiency matters most | SAC, else TD3 | Off-policy data reuse; both are research-grade here |
| You are learning the field | VPG first, then DQN, A2C, PPO, DDPG | Simplest first; complexity added gradually |
| You need a research baseline for TRPO/PPO | OpenAI Baselines, not Spinning Up | Spinning Up's on-policy trio omits observation normalization and normalized value targets |
| You need a research baseline for DDPG/TD3/SAC | Spinning Up is fine | Roughly at parity with best reported results |
| Partial observability or pixels | None of these as shipped | All six are non-recurrent MLP actor-critics |
## Debugging a run that does not learn
1. **Assume it is a bug, not the hyperparameters.** Bad hyperparameters degrade performance;
if yours resemble the paper's, they are probably not the cause.
2. **Check tensor shapes.** A `[N]` vs `[N,1]` mismatch is broadcast-compatible, raises
nothing, and silently corrupts the Bellman backup into an `[N,N]` matrix.
3. **Check what your loss is computed on** — wrong equation, wrong distribution, or data
piped to the wrong place are the three usual causes.
4. **Do not read the policy-gradient loss as a health signal.** Only average return means
anything; the loss can go to negative infinity while performance craters.
5. **Instrument more.** Mean/std/min/max of returns, episode lengths and value estimates;
objective losses; exploration parameters (policy entropy, epsilon). Watch videos too.
6. **Test in more than one environment** once results look promising — code can work in one
environment despite a breaking bug.
## Thresholds and defaults
| Quantity | Value | Source |
|----------|-------|--------|
| Debug-stage turnaround target | **under 5 minutes** locally | Ch 10 |
| Random seeds, minimum | **3** | Ch 10 |
| Random seeds, thorough | **10 or more** | Ch 10, Ch 13 |
| Debug environments | CartPole-v0, InvertedPendulum-v0, FrozenLake-v0, HalfCheetah-v2 at 100-250 steps | Ch 10 |
| From-scratch implementation size | ~250-300 lines; no-frills VPG ~80 | Ch 10 |
| On-policy benchmark network | (64, 32), tanh | Ch 13 |
| Off-policy benchmark network | (256, 256), relu | Ch 13 |
| On-policy batch | 4000 env steps per update | Ch 13 |
| Off-policy minibatch | 100 per gradient step | Ch 13 |
| TD3 `policy_delay` | 2 | Ch 18 |
| Polyak `rho` | in (0,1), usually close to 1 | Ch 17 |
| Benchmark length | 3M timesteps, 5 MuJoCo envs | Ch 13 |
| Plot smoothing used in the book | 11-epoch window | Ch 13 |
## Choosing a research frame
| Frame | Scope | Wraps up in | Main risk |
|-------|-------|-------------|-----------|
| Improve an existing approach | Narrow | A few months | Tweaks fail and you have no signal on what next |
| Unsolved benchmark | Broad | Months to a year-plus | May need a breakthrough; long time, no progress |
| New problem setting | Open-ended | Unbounded | You must design the benchmark too; cannot go looking for these |
## Rigor checklist before reporting
- [ ] Baseline tuned **as much as** your own method. Never handicap it.
- [ ] All else held equal (e.g. comparable parameter counts across architecture variants).
- [ ] At least 3 seeds, ideally 10 or more; plot mean and std dev.
- [ ] Final runs launched fresh and **precommitted** — not the best or most interesting runs.
- [ ] One ablation per design decision, so each claim is separately supported.
- [ ] Checked the idea has not already been published.
## Tells and smells
- **A learning curve with no seed band** is telling you less than it looks like.
- **`AverageEpRet` on an off-policy algorithm** is the noisy behavior policy, not performance.
Use the plotter's `Performance` alias.
- **Evaluating SAC without `--deterministic`** measures the wrong policy.
- **A model-based agent that scores well but behaves badly** is exploiting model bias — the
same shape as a policy exploiting Q-function errors.
- **"Fits the Bellman equation well"** carries no guarantee of good policy performance.
- **A single missing `squeeze`** in a critic is the archetype of silent failure.
- **A paper's full trick list** is usually more than strictly necessary — try simplifications.
- **A library abstraction** is good for reuse across algorithms and a cost when learning one.
## Quick commands
```bash
python -m spinup.run ppo --env Walker2d-v2 --exp_name walker --seed 0 10 20
python -m spinup.run test_policy data/walker/walker_s0 # add -d for SAC
python -m spinup.run plot data/walker # prefix autocompletes
python -m spinup.run [algo] --help # every kwarg is a flag
```

56
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/glossary.md Normal file
View file

@ -0,0 +1,56 @@
# Glossary
**Action space** — the set of all valid actions in an environment; discrete or continuous (Ch 7)
**Action-value function** — see Q-function (Ch 7)
**Advantage function** — `A^pi(s,a) = Q^pi(s,a) - V^pi(s)`; how much better a specific action is than randomly selecting one per pi (Ch 7, Ch 9)
**Agent** — the learner and decider; often used interchangeably with "policy" (Ch 7)
**Backtracking line search** — shrink the step by `alpha^j` until the exact KL constraint holds and surrogate advantage is positive (Ch 15)
**Baseline** — any state-only function `b(s)` subtractable from the policy gradient weight without changing its expectation (Ch 9)
**Bellman backup** — the right-hand side of a Bellman equation: reward plus next value (Ch 7)
**Bellman equation** — self-consistency condition: the value of your starting point is the reward you expect from being there plus the value of where you land next (Ch 7)
**Categorical policy** — stochastic policy for discrete actions, built like a classifier: logits then softmax (Ch 7)
**Clipped double-Q learning** — learn two Q-functions and use the smaller value to form the target (Ch 18, Ch 19)
**Deadly triad** — function approximation + bootstrapping + off-policy data, jointly causing instability in value learning (Ch 8)
**Diagonal Gaussian policy** — stochastic policy for continuous actions; mean network plus a diagonal covariance represented as a vector of log stds (Ch 7)
**Discount factor (gamma)** — in (0,1); makes an infinite reward sum converge and encodes "cash now beats cash later" (Ch 7)
**EGLP lemma** — Expected Grad-Log-Prob: `E_{x~P_theta}[grad log P_theta(x)] = 0` (Ch 9)
**Entropy** — `H(P) = E_{x~P}[-log P(x)]`; how random a random variable is (Ch 19)
**Entropy regularization** — adding an entropy bonus to the reward at each timestep, scaled by `alpha` (Ch 19)
**Epoch** — one experience-collection phase plus one policy update (Ch 9)
**ExperimentGrid** — Spinning Up's hyperparameter ablation tool; backs the CLI (Ch 4, Ch 20)
**Fully / partially observed** — whether the agent sees the complete state or only part of it (Ch 7)
**GAE** — Generalized Advantage Estimation; used by Spinning Up's VPG, TRPO and PPO (Ch 9)
**Log-derivative trick** — `grad P = P * grad log P` (Ch 9)
**Markov property** — transitions depend only on the most recent state and action (Ch 7)
**MDP** — Markov Decision Process, the 5-tuple `<S, A, R, P, rho_0>` (Ch 7)
**Model** — a function predicting state transitions and rewards (Ch 8)
**Model-based / model-free** — whether the agent has or learns a model (Ch 8)
**MPC** — model-predictive control: plan over a window, execute the first action, discard the rest, re-plan (Ch 8)
**MSBE** — Mean-Squared Bellman Error, the loss deep Q-learning minimizes (Ch 17)
**Natural Policy Gradient** — TRPO's analytic solution before the line search is added (Ch 15)
**Observation** — a partial description of a state, possibly omitting information (Ch 7)
**Off-policy** — updates may use data collected at any point in training (Ch 3, Ch 8)
**On-policy** — updates use only data from the most recent policy (Ch 3, Ch 8)
**Optimal policy (pi\*)** — `argmax_pi J(pi)` (Ch 7)
**Parameterized policy** — a policy whose output depends on adjustable parameters, e.g. network weights (Ch 7)
**Performance** — the plotter's family-aware alias, resolving to `AverageEpRet` (on-policy) or `AverageTestEpRet` (off-policy) (Ch 6)
**Policy** — the rule for choosing actions; deterministic `mu(s)` or stochastic `pi(.|s)` (Ch 7)
**Policy delay** — TD3's ratio of Q-updates to policy updates; recommended 2 (Ch 18)
**Policy gradient** — `grad_theta J(pi_theta)` (Ch 9)
**Polyak averaging** — target update `phi_targ <- rho*phi_targ + (1-rho)*phi`; `polyak` in the code (Ch 17)
**Q-function** — `Q^pi(s,a)`, expected return from taking a in s then following pi (Ch 7)
**Replay buffer** — the stored set of past transitions that off-policy algorithms sample from (Ch 17)
**Reparameterization trick** — sample by a deterministic function of state, parameters and independent noise (Ch 19)
**Return** — cumulative reward over a trajectory; finite-horizon undiscounted or infinite-horizon discounted (Ch 7)
**Reward-to-go** — `Rhat_t = sum_{t'=t}^T R(s_t', a_t', s_{t'+1})` (Ch 9)
**Silent failure** — broken RL code that runs correctly and simply never learns (Ch 10, Ch 12)
**Squashed Gaussian policy** — a Gaussian passed through `tanh` to bound actions; SAC's policy (Ch 19)
**start_steps** — initial phase of uniform-random actions used by DDPG, TD3 and SAC (Ch 17)
**State** — a complete description of the world, with nothing hidden (Ch 7)
**Surrogate advantage** — importance-weighted advantage measuring new-policy performance on old-policy data (Ch 15)
**Surrogate objective** — a local approximation of `J(pi_theta)` that is safe to maximize (Ch 8, Ch 16)
**Target network** — a time-lagged copy of a network used to compute stable Bellman targets (Ch 17)
**Target policy smoothing** — clipped noise added to the target action to prevent exploiting sharp Q peaks (Ch 18)
**Trajectory (tau)** — a sequence of states and actions; also called episode or rollout (Ch 7)
**Trust region** — a KL-divergence bound on how far the new policy may move from the old (Ch 15)
**Value function** — `V^pi(s)`, expected return from s following pi (Ch 7)

131
engineering/spinning-up-deep-rl/skills/spinning-up-deep-rl/patterns.md Normal file
View file

@ -0,0 +1,131 @@
# Patterns
Concrete techniques from Spinning Up, each with when to use it, how it works, and what it costs.
## Reward-to-Go Weighting
**When to use** — always, over full-trajectory-return weighting. It is a two-line change.
**How** — weight each `grad log pi(a_t|s_t)` by `sum_{t'=t}^T r_t'` instead of `R(tau)`.
**Trade-offs** — strictly reduces variance at no cost. The dropped terms had zero mean and
nonzero variance, so removing them removes pure noise and cuts the sample trajectories needed.
(Ch 9)
## Value Function Baseline
**When to use** — any policy gradient method. Standard in VPG, TRPO, PPO and A2C.
**How** — subtract `V_phi(s_t)` from the weight; fit `V_phi` by minimizing
`E[(V_phi(s_t) - Rhat_t)^2]` with one or more gradient steps per epoch, starting from the
previous parameters, so it tracks the most recent policy.
**Trade-offs** — provably unbiased by EGLP; reduces variance and gives faster, more stable
learning. Costs a second network and its fitting budget. Under-fitting it is one of the most
drastic performance failures in policy gradients (Exercise 2.1). (Ch 9, Ch 12)
## Trust Region via KL Constraint
**When to use** — when a single bad policy gradient step can collapse performance, and you
can afford second-order machinery.
**How** — maximize surrogate advantage subject to `Dbar_KL <= delta`; Taylor-expand, solve
analytically via Lagrangian duality, then backtracking line search until the exact KL
constraint holds and surrogate advantage is positive. Use conjugate gradient for `H^{-1}g`.
**Trade-offs** — buys monotone-ish, fast improvement and larger safe steps. Costs complexity,
Hessian-vector products, and (in Spinning Up) no PyTorch implementation. (Ch 15)
## Clipped Surrogate Objective
**When to use** — as the first-order replacement for a trust region. The default modern choice.
**How** — `L = min(ratio * A, g(eps, A))` where `g` caps at `(1+eps)A` for positive advantage
and `(1-eps)A` for negative. Take multiple minibatch SGD steps per batch.
**Trade-offs** — much simpler than TRPO and empirically at least as good, but it removes the
*incentive* to move far rather than guaranteeing you do not. Pair it with KL early stopping.
(Ch 16)
## KL Early Stopping
**When to use** — alongside clipping, whenever you take many gradient steps per batch.
**How** — if the mean KL-divergence of the new policy from the old exceeds a threshold, stop
taking gradient steps for this batch.
**Trade-offs** — one extra measurement per step; catches the drift the clip alone allows.
Other PPO implementations use different tricks here. (Ch 16)
## Experience Replay Buffer
**When to use** — every off-policy value-learning algorithm.
**How** — store `(s,a,r,s',d)` tuples; sample minibatches uniformly for MSBE minimization.
Licensed by the fact that the Bellman equation is indifferent to how transitions were collected.
**Trade-offs** — wrong in both directions. Too small and you overfit to the most recent data
and things break; too large and learning slows. It needs tuning. (Ch 17)
## Target Networks
**When to use** — any MSBE minimization, because the target otherwise depends on the
parameters being trained.
**How** — keep a lagged copy. DQN-style: copy every fixed number of steps. DDPG-style:
polyak average once per main update, `phi_targ <- rho*phi_targ + (1-rho)*phi`, `rho` near 1.
**Trade-offs** — stabilizes learning; adds a network and a hyperparameter, and slows the
propagation of new information into the target. (Ch 17)
## Amortizing a Continuous Argmax into a Policy
**When to use** — an inner `max` over a continuous variable inside a loop you run constantly.
**How** — assume differentiability with respect to that variable, learn `mu_theta(s)` by
gradient ascent on `Q_phi(s, mu_theta(s))`, and substitute `max_a Q(s,a) ~= Q(s, mu(s))`.
**Trade-offs** — turns an intractable per-step optimization into a forward pass. Costs an
approximation whose errors the policy will actively exploit. (Ch 17)
## Clipped Double-Q
**When to use** — whenever a learned Q-function is being maximized over by a policy, i.e.
the whole DDPG family.
**How** — learn two Q-functions, use `min` of the two target values as the shared regression
target for both.
**Trade-offs** — fends off overestimation, which is DDPG's dominant failure. Costs a second
critic and introduces some underestimation bias. TD3 uses `Q_phi_1` alone in the policy loss;
SAC uses the `min` there too. (Ch 18, Ch 19)
## Delayed Policy Updates
**When to use** — when policy updates destabilize the Bellman target.
**How** — update the policy and all target networks once per `policy_delay` critic updates;
the paper recommends 2.
**Trade-offs** — damps volatility; slows policy improvement per environment step. (Ch 18)
## Target Policy Smoothing
**When to use** — deterministic policies over continuous actions.
**How** — `a'(s') = clip(mu_targ(s') + clip(eps, -c, c), a_Low, a_High)`, `eps ~ N(0, sigma)`.
**Trade-offs** — regularizes Q along the action dimension so a spurious sharp peak cannot be
exploited. A stochastic policy (SAC) gets a similar effect for free. (Ch 18, Ch 19)
## Entropy Regularization
**When to use** — when premature convergence to a bad local optimum is the risk, or when you
want an explicit explore-exploit dial.
**How** — add `alpha * H(pi(.|s_t))` to the reward at each timestep; propagate the term into
the value functions and the Bellman target.
**Trade-offs** — higher `alpha` means more exploration and faster later learning; the right
value is environment-specific and needs careful tuning. The entropy-constrained variant that
adapts `alpha` is generally preferred by practitioners over the fixed one. (Ch 19)
## Reparameterization Trick for Stochastic Policies
**When to use** — differentiating through an expectation whose distribution depends on the
parameters.
**How** — sample by a deterministic function of state, parameters and independent noise:
`a~ = tanh(mu_theta(s) + sigma_theta(s) * xi)`, `xi ~ N(0,I)`, converting the expectation over
actions into an expectation over noise.
**Trade-offs** — makes the policy differentiable end to end. The `tanh` squash bounds actions
but changes the distribution; log-probabilities are still closed-form. (Ch 19)
## Uniform Random Warm-Up (start_steps)
**When to use** — every off-policy algorithm here, at the beginning of training.
**How** — for a fixed number of steps take actions sampled uniformly over valid actions, then
switch to normal exploration.
**Trade-offs** — fills the replay buffer with diverse data before the policy can bias it.
Wasted steps if the environment is expensive. (Ch 17, Ch 18, Ch 19)
## Multi-Seed Reporting with Variance Bands
**When to use** — every reported RL result.
**How** — at least 3 seeds, 10 or more to be thorough; plot the mean as a solid line and the
standard deviation as a shaded band; smooth for display only.
**Trade-offs** — multiplies compute by the seed count. Without it, deep RL's seed sensitivity
means two seed groups can look like different distributions entirely. (Ch 10, Ch 13)
## Precommitted Final Runs
**When to use** — before reporting any comparison.
**How** — use the tuning stage to form hypotheses, then launch fresh final experiments for
every method compared and commit in advance to reporting whatever comes out.
**Trade-offs** — a weak form of preregistration; costs one more full experiment round and
removes the largest source of accidental self-deception. (Ch 10)
## Per-Claim Ablation
**When to use** — any method with more than one design decision, which is all of them.
**How** — swap out or remove each design element separately and measure.
**Trade-offs** — turns one bundled claim into several separately-confident ones. Costs one
experiment per element. Also the fastest way to learn which parts you can delete. (Ch 10)

2
mkdocs.yml
View file

@ -1,6 +1,6 @@
site_name: Claude Code Skills & Agent Plugins
site_url: https://alirezarezvani.github.io/claude-skills/
site_description: "380 production-ready agent skills, 96 installable plugins, and 138 slash commands across 20 domains — engineering, product, marketing, compliance, finance, research, and agent tooling. Works with Claude Code, OpenAI Codex, Gemini CLI, Cursor, Hermes Agent, Mistral Vibe, OpenClaw, and 6 more AI coding tools. Open source, MIT licensed, zero dependencies."
site_description: "388 production-ready skills across 20 domains, 99 marketplace plugins, and 150 slash commands — engineering, product, marketing, compliance, finance, research, and agent tooling. Works with Claude Code, OpenAI Codex, Gemini CLI, Cursor, Hermes Agent, Mistral Vibe, OpenClaw, and 6 more AI coding tools. Open source, MIT licensed, zero dependencies."
site_author: Alireza Rezvani
repo_url: https://github.com/alirezarezvani/claude-skills
repo_name: alirezarezvani/claude-skills

10
research/pulse/README.md
View file

@ -8,7 +8,7 @@ This is the **research-pack shape** anchor — its Agent Integrity Rules block t
1. **Grill-me intake** — 2–4 forcing questions, one at a time: topic specificity, angle (trend/sentiment/problems/opportunities/comparison), time window (7d/14d/30d/60d/90d), platform scope.
2. **Parallel Phases 1–3** — Reddit + Hacker News + open web fire concurrently. 1 q/sec rate limit per platform; sequential calls within each platform.
3. **Optional Phase 4** — X/Twitter via Grok / X API / browser automation if available. Skipped with note otherwise.
3. **Optional Phase 4** — normalize a supplied X export first. Fall back to Grok, X API, or browser automation.
4. **Synthesis** — cross-platform pattern detection: consensus, controversy, pain points, excitement, emerging trends, gaps.
5. **Output** — markdown file at `${RESEARCH_DIR}/pulse/<topic-slug>-<YYYY-MM-DD>.md` AND full briefing in chat.
@ -31,7 +31,7 @@ The skill is **recency-oriented** — it captures the current conversation, not
|---|---|
| `skills/pulse/SKILL.md` | The skill itself (Claude reads this when triggered) |
| `skills/pulse/scripts/time_window_calculator.py` | Deterministic Unix-timestamp + Reddit `t=` parameter computation from window string |
| `skills/pulse/scripts/citation_tracker.py` | JSON-backed three-count audit log (sent / received / cited) |
| `skills/pulse/scripts/citation_tracker.py` | Three-count audit log plus local X export normalization and deduplication |
| `skills/pulse/scripts/topic_slug_generator.py` | Filesystem-safe slug + duplicate-date detection for output paths |
| `skills/pulse/references/research_pack_conventions.md` | The Agent Integrity Rules canon (7+ sources) |
| `skills/pulse/references/cross_platform_synthesis.md` | Consensus/controversy/pain detection across platforms (7+ sources) |
@ -48,6 +48,12 @@ python skills/pulse/scripts/time_window_calculator.py --window 30d
# Start a citation tracker session
python skills/pulse/scripts/citation_tracker.py --action start --session pulse-2026-05-15-claude-code
# Import an existing Xquik, X API v2, or generic X search export
python skills/pulse/scripts/citation_tracker.py --action import_sources \
--session pulse-2026-05-15-claude-code \
--input /path/to/x-search.json \
--platform x
# Generate the output-file slug for a topic
python skills/pulse/scripts/topic_slug_generator.py --topic "self-hosted LLM deployment" --date 2026-05-15
```

17
research/pulse/agents/cs-pulse.md
View file

@ -35,7 +35,7 @@ The cs-pulse agent orchestrates the `pulse` skill across multi-source recency br
1. **Grill-me intake (Q1 → Q4, dependency-ordered)** — topic, angle, window, scope. One at a time. Refuse vague answers.
2. **Pre-flight** — compute window timestamps with `skills/pulse/scripts/time_window_calculator.py`, generate output slug with `skills/pulse/scripts/topic_slug_generator.py`, start three-count audit with `skills/pulse/scripts/citation_tracker.py`.
3. **Phases 1–3 in parallel** — Reddit (top + new), HN (Algolia stories + comments), Web (2–3 targeted queries). 1 q/sec per platform; sequential within.
4. **Phase 4 (optional)** — X/Twitter if available; skip with note otherwise.
4. **Phase 4 (optional)** — normalize a supplied X export first. Try a live interface only when needed.
5. **Synthesis** — cross-platform pattern detection (consensus, controversy, pain, excitement, gaps).
6. **Output** — save file + paste full briefing in chat.
@ -69,8 +69,8 @@ Differentiates clearly:
2. **Citation Tracker**
- Path: `../skills/pulse/scripts/citation_tracker.py`
- Usage: `python citation_tracker.py --action {start,record_sent,record_received,record_cited,status,close} --session NAME`
- JSON-backed audit log at `~/.pulse_sessions/<session>.json`. Each call increments the three counts. Output the audit summary block for the synthesis section.
- Usage: `python citation_tracker.py --action {start,record_sent,record_received,record_cited,import_sources,status,close} --session NAME`
- Tracks the three counts. `import_sources` normalizes local Xquik, X API v2, or generic JSON and deduplicates Tweet IDs.
3. **Topic Slug Generator**
- Path: `../skills/pulse/scripts/topic_slug_generator.py`
@ -101,7 +101,9 @@ python ../skills/pulse/scripts/citation_tracker.py --action start --session "pul
python ../skills/pulse/scripts/citation_tracker.py --action record_sent --session NAME --query "..."
python ../skills/pulse/scripts/citation_tracker.py --action record_received --session NAME --count N
# C. Phase 4 (optional): X/Twitter via Grok / X API / browser automation. Skip with note if unavailable.
# C. Phase 4 (optional): import a supplied export before using a live interface.
python ../skills/pulse/scripts/citation_tracker.py --action import_sources \
--session NAME --input /path/to/x-search.json --platform x
# D. Synthesis — cross-platform pattern detection. For each cited source:
python ../skills/pulse/scripts/citation_tracker.py --action record_cited --session NAME --url "https://..."
@ -123,9 +125,10 @@ python ../skills/pulse/scripts/citation_tracker.py --action close --session NAME
| Context | Phase 4 behavior |
|---|---|
| Claude Code CLI with browser automation | Run X/Twitter via Grok or available interface |
| Claude Code CLI without browser automation | Skip Phase 4 with documented note in output |
| Claude.ai web | Skip Phase 4 (browser automation unavailable); note in output |
| Supplied local X export | Normalize, filter, deduplicate, and analyze it without network access |
| Claude Code CLI with browser automation | Use a live interface only when no export was supplied |
| Claude Code CLI without browser automation | Skip Phase 4 when no export was supplied |
| Claude.ai web | Analyze an attached export. Otherwise skip Phase 4. |
| Any context | Phases 1–3 always run |
## Output Standards

10
research/pulse/commands/cs-pulse.md
View file

@ -75,7 +75,7 @@ Saved to `${RESEARCH_DIR}/pulse/<topic-slug>-<YYYY-MM-DD>.md` AND pasted in chat
- **Source discipline** — cite only session-call results. `[Background]` for training knowledge, excluded from cited count.
- **Three-count tracking** — sent / received / cited in audit log.
- **Retry once after 3s** — then log. 3 consecutive failures across sources → stop.
- **Graceful degradation** — single source failure → continue with rest. Never fail the whole run on one source.
- **Graceful degradation** — prefer a supplied X export. Skip only when no export or live interface exists.
## Workflow
@ -90,7 +90,10 @@ python ../skills/pulse/scripts/citation_tracker.py --action start --session NAME
# HN: Algolia stories + comments with timestamp filter
# Web: 2–3 targeted queries
# C. Phase 4 (optional, sequential): X/Twitter via Grok / X API / browser automation
# C. Phase 4 (optional, sequential): normalize a supplied export first
python ../skills/pulse/scripts/citation_tracker.py --action import_sources \
--session NAME --input /path/to/x-search.json --platform x
# If no export exists, try Grok / X API / browser automation.
# D. Synthesis: cross-platform pattern detection
@ -109,8 +112,9 @@ python ../skills/pulse/scripts/citation_tracker.py --action close --session NAME
- Starting any search before Q1 (topic specificity) commits
- Batching intake questions
- Hardcoded URLs that won't survive API changes (note format, explain may evolve)
- Specific person/brand references
- Irrelevant person or brand references
- Tight coupling to one X/Twitter interface
- Treating duplicate Tweet IDs or repeated citation URLs as separate sources
- Missing fallback behavior
- "Just use [specific tool]" without explaining what the tool does
- Citing training knowledge as session results

33
research/pulse/skills/pulse/SKILL.md
View file

@ -11,7 +11,7 @@ metadata:
# Pulse — Multi-Source Recency Research
> **Portability:** Works in both Claude Code CLI and Claude.ai. The optional X/Twitter phase requires browser automation and is skipped automatically if unavailable.
> **Portability:** Works in Claude Code CLI and Claude.ai. Phase 4 accepts a local X/Twitter search export before trying a live interface.
A recency-oriented research skill that synthesizes what people are saying about a topic across Reddit, Hacker News, the open web, and (optionally) X/Twitter — within a configurable time window. Output is a single coherent briefing with citations, engagement signals, and cross-platform pattern analysis. The skill captures the **current conversation**, not the canonical reference.
@ -138,10 +138,25 @@ Run last. Reasons:
- X content overlaps significantly with Reddit/HN — so it adds delta, not primary signal
**Interface (in priority order):**
1. **Grok** if available in the harness
2. **X API** if authenticated
3. **Browser automation** if the harness supports it (Claude Code CLI with `playwright` or similar)
4. **Skip with note** if none of the above available
1. **User-provided JSON export.** Import it before any live request:
```bash
python3 scripts/citation_tracker.py \
--action import_sources \
--session NAME \
--input /path/to/x-search.json \
--platform x \
--since 2026-07-01T00:00:00Z \
--until 2026-08-01T00:00:00Z
```
The importer accepts Xquik Tweet Search, X API v2, and generic JSON exports.
It normalizes legacy and snake-case fields, joins X API `includes.users`,
filters the requested window, and deduplicates by Tweet ID. It makes no
network calls and requires no API key. The audit stores the filename and
SHA-256 digest, not the user's absolute path.
2. **Grok** if available in the harness.
3. **X API** if authenticated.
4. **Browser automation** if the harness supports it.
5. **Skip with note** if none of the above are available.
**Documented behavior:**
> If Phase 4 is skipped: include the section header `## X/Twitter` with body `Skipped — [reason: no browser automation / no Grok / no X API]`. Do NOT pretend to have data.
@ -220,7 +235,8 @@ Sources received: M. Sources cited: K. Training knowledge: 0 ([Background] exclu
| Reddit blocks / rate-limits | Try `?raw_json=1` or fall back to subreddit-restricted search. Honor 3s-retry. |
| HN returns empty | Broaden query, drop timestamp filter as last resort, label results "outside window". |
| Web search returns nothing useful | Note in output; don't fabricate sources. |
| Browser automation unavailable | Skip Phase 4 with documented note. |
| Browser automation unavailable | Import a supplied export. Otherwise skip Phase 4 with a note. |
| Local X export is invalid | Stop Phase 4. Report the parse error. Do not guess missing records. |
| WebFetch times out | Use what loaded, mark the source as "truncated". |
| 3 consecutive failures across sources | Stop. Return what was collected with explicit "stopped early" note. Do NOT deliver empty file. |
| All sources fail | Return error with diagnostic info. Do NOT deliver empty file. |
@ -230,7 +246,7 @@ Sources received: M. Sources cited: K. Training knowledge: 0 ([Background] exclu
| Script | Role |
|---|---|
| `scripts/time_window_calculator.py` | Compute Unix timestamps + Reddit `t=` parameter from window string (`30d`, `7d`, etc.). Deterministic from `datetime.now()`. |
| `scripts/citation_tracker.py` | JSON-backed three-count audit log (sent / received / cited) at `~/.pulse_sessions/<session>.json`. |
| `scripts/citation_tracker.py` | Three-count audit log plus local X export normalization and deduplication. |
| `scripts/topic_slug_generator.py` | Filesystem-safe slug + duplicate-date detection for output paths. |
## References
@ -244,8 +260,9 @@ Sources received: M. Sources cited: K. Training knowledge: 0 ([Background] exclu
- Starting any search before the user commits to topic specificity (Q1)
- Batching intake questions instead of one at a time
- Hardcoded URLs that won't survive API changes (note format, explain may evolve)
- Specific person / brand references in the skill body
- Irrelevant person or brand references in the skill body
- Tight coupling to one X/Twitter interface
- Counting duplicate Tweet IDs or repeated citation URLs as separate sources
- Missing fallback behavior on source failure
- "Just use [specific tool]" without explaining what the tool does
- Citing training knowledge in the cited count

334
research/pulse/skills/pulse/scripts/citation_tracker.py
View file

@ -5,7 +5,7 @@ Stdlib-only. Maintains the research-pack convention's three counts:
- queries sent (every tool call issued)
- sources received (every item returned across all queries)
- sources cited (every URL that made it into the final synthesis)
- sources cited (every unique URL in the final synthesis)
Session state persists in ~/.pulse_sessions/<session>.json so runs can be
inspected and resumed.
@ -17,6 +17,7 @@ Actions:
record_sent Increment sent count + log the query
record_received Increment received count by N
record_cited Increment cited count + log the URL
import_sources Normalize a local X export and record unique sources
status Show current counts + audit summary block
list List existing sessions
close Finalize the session (set ended_at timestamp)
@ -26,21 +27,53 @@ Usage:
python citation_tracker.py --action record_sent --session pulse-... --query "claude code adoption" --platform reddit
python citation_tracker.py --action record_received --session pulse-... --count 12 --platform reddit
python citation_tracker.py --action record_cited --session pulse-... --url "https://reddit.com/..." --platform reddit
python citation_tracker.py --action import_sources --session pulse-... --input x-search.json --platform x
python citation_tracker.py --action status --session pulse-...
python citation_tracker.py --action list
python citation_tracker.py --action close --session pulse-...
"""
import argparse
import hashlib
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Tuple
SESSIONS_DIR = Path.home() / ".pulse_sessions"
IMPORT_PROVIDERS = ("auto", "generic", "x-api-v2", "xquik")
SAMPLE_XQUIK_EXPORT: Dict[str, Any] = {
"tweets": [
{
"id": "100",
"text": "Public launch feedback",
"createdAt": "2026-08-20T10:00:00Z",
"likeCount": 7,
"retweetCount": 2,
"replyCount": 1,
"author": {"id": "10", "username": "example"},
},
{
"id": "100",
"text": "Public launch feedback",
"createdAt": "2026-08-20T10:00:00Z",
"author": {"id": "10", "username": "example"},
},
{
"id": "101",
"text": "A second public response",
"created_at": 1787223600,
"author_username": "second_example",
"public_metrics": {"like_count": 3, "repost_count": 1},
},
],
"has_more": False,
"next_cursor": "",
}
def session_path(name: str) -> Path:
@ -63,6 +96,205 @@ def now_iso() -> str:
return datetime.now(timezone.utc).isoformat()
def first_value(data: Dict[str, Any], names: Tuple[str, ...]) -> Any:
for name in names:
value = data.get(name)
if value is not None:
return value
return None
def parse_timestamp(value: Any) -> Optional[datetime]:
if value is None or value == "":
return None
if isinstance(value, (int, float)):
return datetime.fromtimestamp(value, timezone.utc)
if not isinstance(value, str):
raise ValueError(f"unsupported timestamp {value!r}")
normalized = value[:-1] + "+00:00" if value.endswith("Z") else value
parsed = datetime.fromisoformat(normalized)
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
def integer_value(data: Dict[str, Any], names: Tuple[str, ...]) -> int:
value = first_value(data, names)
if value is None:
return 0
try:
return max(0, int(value))
except (TypeError, ValueError):
return 0
def detect_provider(payload: Any) -> str:
if isinstance(payload, dict):
if isinstance(payload.get("tweets"), list):
return "xquik"
if isinstance(payload.get("data"), list):
return "x-api-v2"
return "generic"
def extract_rows(payload: Any, provider: str) -> List[Dict[str, Any]]:
if isinstance(payload, list):
rows = payload
elif isinstance(payload, dict):
if provider == "xquik":
if not isinstance(payload.get("tweets"), list):
raise ValueError("Xquik import must contain a tweets array")
rows = payload["tweets"]
elif provider == "x-api-v2":
if not isinstance(payload.get("data"), list):
raise ValueError("X API v2 import must contain a data array")
rows = payload["data"]
elif first_value(payload, ("id", "tweet_id", "tweetId", "id_str")) is not None:
rows = [payload]
else:
key = next(
(candidate for candidate in ("records", "items", "results", "tweets", "data")
if isinstance(payload.get(candidate), list)),
"",
)
if not key:
raise ValueError("generic import must be a Tweet object, array, or known list container")
rows = payload[key]
else:
raise ValueError("import root must be a JSON object or array")
return [row for row in rows if isinstance(row, dict)]
def x_api_users(payload: Any) -> Dict[str, Dict[str, Any]]:
if not isinstance(payload, dict):
return {}
includes = payload.get("includes")
if not isinstance(includes, dict) or not isinstance(includes.get("users"), list):
return {}
return {
str(user["id"]): user
for user in includes["users"]
if isinstance(user, dict) and user.get("id") is not None
}
def normalize_row(row: Dict[str, Any], users: Dict[str, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
tweet_id = first_value(row, ("id", "tweet_id", "tweetId", "id_str"))
text = first_value(row, ("text", "full_text", "fullText"))
if tweet_id is None or not isinstance(text, str) or not text.strip():
return None
embedded_author = row.get("author") if isinstance(row.get("author"), dict) else {}
author_id = first_value(row, ("author_id", "authorId")) or first_value(
embedded_author, ("id", "id_str")
)
included_author = users.get(str(author_id), {}) if author_id is not None else {}
username = first_value(row, ("author_username", "authorUsername", "username"))
if username is None:
username = first_value(embedded_author, ("username", "screen_name", "screenName"))
if username is None:
username = first_value(included_author, ("username", "screen_name"))
metrics = row.get("public_metrics") if isinstance(row.get("public_metrics"), dict) else row
identifier = str(tweet_id)
source_url = first_value(row, ("url", "permalink", "tweet_url", "tweetUrl"))
if not source_url:
account = str(username).lstrip("@") if username else "i/web"
source_url = f"https://x.com/{account}/status/{identifier}"
return {
"id": identifier,
"url": str(source_url),
"text": text.strip(),
"created_at": first_value(row, ("created_at", "createdAt")),
"author": {
"id": str(author_id) if author_id is not None else None,
"username": str(username).lstrip("@") if username else None,
},
"metrics": {
"likes": integer_value(metrics, ("like_count", "likeCount", "favorite_count")),
"reposts": integer_value(
metrics, ("repost_count", "retweet_count", "retweetCount")
),
"replies": integer_value(metrics, ("reply_count", "replyCount")),
"quotes": integer_value(metrics, ("quote_count", "quoteCount")),
"views": integer_value(metrics, ("view_count", "viewCount", "impression_count")),
"bookmarks": integer_value(metrics, ("bookmark_count", "bookmarkCount")),
},
}
def normalize_export(
payload: Any,
provider: str = "auto",
since: Optional[str] = None,
until: Optional[str] = None,
) -> Dict[str, Any]:
selected_provider = detect_provider(payload) if provider == "auto" else provider
rows = extract_rows(payload, selected_provider)
users = x_api_users(payload)
since_at = parse_timestamp(since)
until_at = parse_timestamp(until)
if since_at and until_at and since_at >= until_at:
raise ValueError("--since must be earlier than --until")
sources: List[Dict[str, Any]] = []
seen_ids = set()
duplicates = 0
filtered = 0
malformed = 0
for row in rows:
source = normalize_row(row, users)
if source is None:
malformed += 1
continue
if source["id"] in seen_ids:
duplicates += 1
continue
seen_ids.add(source["id"])
if since_at or until_at:
try:
created_at = parse_timestamp(source.get("created_at"))
except ValueError:
malformed += 1
continue
if created_at is None:
malformed += 1
continue
if since_at and created_at < since_at:
filtered += 1
continue
if until_at and created_at >= until_at:
filtered += 1
continue
sources.append(source)
return {
"provider": selected_provider,
"input_count": len(rows),
"accepted_count": len(sources),
"duplicate_count": duplicates,
"filtered_count": filtered,
"malformed_count": malformed,
"sources": sources,
}
def load_export(path: str) -> Tuple[Any, Dict[str, str]]:
input_path = Path(path).expanduser()
try:
raw = input_path.read_text(encoding="utf-8")
payload = json.loads(raw)
except OSError as exc:
raise ValueError(f"cannot read import: {exc}") from exc
except json.JSONDecodeError as exc:
raise ValueError(f"import is not valid JSON: {exc}") from exc
return payload, {
"input_name": input_path.name,
"input_sha256": hashlib.sha256(raw.encode("utf-8")).hexdigest(),
}
def action_start(name: str, topic: Optional[str]) -> Dict[str, Any]:
if session_path(name).exists():
raise FileExistsError(f"Session already exists: {name}")
@ -74,6 +306,8 @@ def action_start(name: str, topic: Optional[str]) -> Dict[str, Any]:
"queries_sent": [],
"sources_received": [],
"sources_cited": [],
"source_imports": [],
"imported_sources": [],
"counts": {"sent": 0, "received": 0, "cited": 0},
}
save_session(name, data)
@ -89,6 +323,8 @@ def action_record_sent(name: str, query: str, platform: str) -> Dict[str, Any]:
def action_record_received(name: str, count: int, platform: str) -> Dict[str, Any]:
if count < 0:
raise ValueError("--count cannot be negative")
data = load_session(name)
data["sources_received"].append({"count": count, "platform": platform, "at": now_iso()})
data["counts"]["received"] += count
@ -98,12 +334,51 @@ def action_record_received(name: str, count: int, platform: str) -> Dict[str, An
def action_record_cited(name: str, url: str, platform: str) -> Dict[str, Any]:
data = load_session(name)
if any(source.get("url") == url for source in data["sources_cited"]):
return data
data["sources_cited"].append({"url": url, "platform": platform, "at": now_iso()})
data["counts"]["cited"] += 1
save_session(name, data)
return data
def action_import_sources(
name: str,
input_path: str,
platform: str,
provider: str,
since: Optional[str],
until: Optional[str],
) -> Dict[str, Any]:
payload, provenance = load_export(input_path)
report = normalize_export(payload, provider, since, until)
data = load_session(name)
imported_sources = data.setdefault("imported_sources", [])
existing_ids = {source.get("id") for source in imported_sources}
new_sources = [source for source in report["sources"] if source["id"] not in existing_ids]
imported_sources.extend(new_sources)
data.setdefault("source_imports", []).append({
**provenance,
"platform": platform,
"provider": report["provider"],
"input_count": report["input_count"],
"accepted_count": len(new_sources),
"duplicate_count": report["duplicate_count"] + len(report["sources"]) - len(new_sources),
"filtered_count": report["filtered_count"],
"malformed_count": report["malformed_count"],
"at": now_iso(),
})
data["sources_received"].append({
"count": len(new_sources),
"platform": platform,
"kind": "local_import",
"at": now_iso(),
})
data["counts"]["received"] += len(new_sources)
save_session(name, data)
return data
def action_status(name: str) -> Dict[str, Any]:
return load_session(name)
@ -155,6 +430,15 @@ def render_status_human(data: Dict[str, Any]) -> str:
out.append("Sent by platform:")
for plat, n in sorted(by_platform_sent.items(), key=lambda kv: -kv[1]):
out.append(f" {plat:<10s} {n}")
imports = data.get("source_imports", [])
if imports:
out.append("Local imports:")
for item in imports:
out.append(
f" {item.get('input_name', '(unknown)')}: "
f"{item.get('accepted_count', 0)}/{item.get('input_count', 0)} accepted "
f"({item.get('provider', 'generic')})"
)
out.append("")
out.append("Audit block (paste in synthesis):")
parts: List[str] = []
@ -188,8 +472,10 @@ def main(argv: List[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n")[0])
parser.add_argument(
"--action",
choices=["start", "record_sent", "record_received", "record_cited", "status", "list", "close"],
required=True,
choices=[
"start", "record_sent", "record_received", "record_cited",
"import_sources", "status", "list", "close",
],
)
parser.add_argument("--session", help="Session name")
parser.add_argument("--topic", help="(start only) topic string")
@ -197,9 +483,37 @@ def main(argv: List[str]) -> int:
parser.add_argument("--platform", help="(record_* only) platform name: reddit | hn | web | x | other")
parser.add_argument("--count", type=int, help="(record_received only) number of sources received")
parser.add_argument("--url", help="(record_cited only) cited URL")
parser.add_argument("--input", help="(import_sources only) local JSON export")
parser.add_argument(
"--provider", choices=IMPORT_PROVIDERS, default="auto",
help="(import_sources only) input schema; default: auto",
)
parser.add_argument("--since", help="(import_sources only) inclusive ISO timestamp")
parser.add_argument("--until", help="(import_sources only) exclusive ISO timestamp")
parser.add_argument("--sample", action="store_true", help="normalize a built-in Xquik sample")
parser.add_argument("--output", choices=["human", "json"], default="human")
parser.add_argument("--json", action="store_true", help="alias for --output json")
args = parser.parse_args(argv)
output = "json" if args.json else args.output
if args.sample:
result = normalize_export(
SAMPLE_XQUIK_EXPORT,
since="2026-08-20T00:00:00Z",
until="2026-08-21T00:00:00Z",
)
if output == "json":
print(json.dumps(result, indent=2))
else:
print(
f"Provider: {result['provider']}\n"
f"Accepted: {result['accepted_count']}\n"
f"Duplicates: {result['duplicate_count']}"
)
return 0
if not args.action:
parser.error("--action is required unless --sample is used")
try:
if args.action == "start":
if not args.session:
@ -221,6 +535,14 @@ def main(argv: List[str]) -> int:
print("error: --session, --url, --platform required for record_cited", file=sys.stderr)
return 2
result = action_record_cited(args.session, args.url, args.platform)
elif args.action == "import_sources":
if not (args.session and args.input):
print("error: --session and --input required for import_sources", file=sys.stderr)
return 2
result = action_import_sources(
args.session, args.input, args.platform or "x", args.provider,
args.since, args.until,
)
elif args.action == "status":
if not args.session:
print("error: --session required for status", file=sys.stderr)
@ -233,11 +555,11 @@ def main(argv: List[str]) -> int:
result = action_close(args.session)
else: # list
result = action_list()
except (FileNotFoundError, FileExistsError) as e:
except (FileNotFoundError, FileExistsError, ValueError) as e:
print(f"error: {e}", file=sys.stderr)
return 2
if args.output == "json":
if output == "json":
print(json.dumps(result, indent=2, default=str))
else:
if args.action == "list":

8
scripts/check_model_freshness.py
View file

@ -35,7 +35,9 @@ import os
import re
import sys
EXCLUDED_DIRS = {
# Excludes both directory names (pruned during walk) and individual filenames
# (e.g. CHANGELOG.md) — membership is checked against dirnames AND filenames.
EXCLUDED_NAMES = {
".git", ".codex", ".gemini", ".hermes", ".vibe", "node_modules",
"docs", # generated mirror; fix the source instead
"audit", # audit records quote stale IDs on purpose, that is their job
@ -157,9 +159,9 @@ def scan_file(path, repo_root, allowlist):
def collect(repo_root):
targets = []
for dirpath, dirnames, filenames in os.walk(repo_root):
dirnames[:] = [d for d in dirnames if d not in EXCLUDED_DIRS]
dirnames[:] = [d for d in dirnames if d not in EXCLUDED_NAMES]
for fn in filenames:
if fn in EXCLUDED_DIRS or fn in SELF_FILES:
if fn in EXCLUDED_NAMES or fn in SELF_FILES:
continue
if fn.endswith(SCAN_EXTENSIONS):
targets.append(os.path.join(dirpath, fn))

44
scripts/derive_counters.py
View file

@ -3,7 +3,7 @@
Walks the canonical tree (excluding sync copies, docs site, audit workspace,
and VCS/CI internals) and derives the headline numbers that README.md,
CLAUDE.md, and .claude-plugin/marketplace.json claim:
CLAUDE.md, marketplace.json, mkdocs.yml, and .codex-plugin/plugin.json claim:
skills count of SKILL.md files
plugins_on_disk count of **/.claude-plugin/plugin.json manifests
@ -19,11 +19,13 @@ Modes:
(default) print a human-readable table
--json print the derived counters as JSON
--check exit 1 listing mismatches if the headline counters claimed in
README.md, root CLAUDE.md ("Current Scope" line), and
marketplace.json metadata.description disagree with derived
values. Also validates the README "Skills Overview" per-domain
table: every domain row's count must equal the SKILL.md count in
its linked folder, and every on-disk domain must have a row. CI gate G3.
README.md, root CLAUDE.md ("Current Scope" / "Status:" lines),
marketplace.json metadata.description, mkdocs.yml
site_description, or .codex-plugin/plugin.json descriptions
disagree with derived values. Also validates the README
"Skills Overview" per-domain table: every domain row's count
must equal the SKILL.md count in its linked folder, and every
on-disk domain must have a row. CI gate G3.
Stdlib only. No writes ever.
"""
@ -298,6 +300,32 @@ def run_check(root: Path, derived: dict) -> int:
print(f"FAIL: cannot parse marketplace.json: {exc}")
return 1
# The two sites PR #940 found drifting ungated: the docs-site description
# and the Codex plugin manifest (which had lagged nine releases behind).
mkdocs = root / "mkdocs.yml"
if mkdocs.is_file():
# mkdocs.yml carries !!python tags safe_load rejects — read as text and
# restrict to the site_description line, mirroring the CLAUDE.md approach.
desc_lines = [ln for ln in mkdocs.read_text(encoding="utf-8").splitlines()
if ln.startswith("site_description:")]
sources.append(("mkdocs.yml (site_description)", "\n".join(desc_lines)))
codex_manifest = root / ".codex-plugin" / "plugin.json"
if codex_manifest.is_file():
try:
data = json.loads(codex_manifest.read_text(encoding="utf-8"))
# Include the interface descriptions too — they carry their own
# counts; only standardized-phrasing claims in them are gated.
iface = data.get("interface", {})
codex_text = " ".join(str(s) for s in (
data.get("description", ""),
iface.get("shortDescription", ""),
iface.get("longDescription", "")))
sources.append((".codex-plugin/plugin.json descriptions", codex_text))
except (json.JSONDecodeError, OSError) as exc:
print(f"FAIL: cannot parse .codex-plugin/plugin.json: {exc}")
return 1
mismatches = []
for label, text in sources:
claims = extract_claims(text)
@ -322,7 +350,7 @@ def run_check(root: Path, derived: dict) -> int:
print("\nRun `python3 scripts/derive_counters.py` for the ground-truth table.")
return 1
print("Counter check passed: README.md, CLAUDE.md, marketplace.json match derived values.")
print("Counter check passed: README.md, CLAUDE.md, marketplace.json, mkdocs.yml, .codex-plugin match derived values.")
return 0
@ -334,7 +362,7 @@ def main() -> int:
parser.add_argument(
"--check",
action="store_true",
help="exit 1 if README.md / CLAUDE.md / marketplace.json claims drift from derived values",
help="exit 1 if claims in README.md / CLAUDE.md / marketplace.json / mkdocs.yml / .codex-plugin drift from derived values",
)
args = parser.parse_args()